~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* Per core/cpu state
3	4	*
..	..	@@ -17,7 +18,9 @@
17	18	#include <asm/cpufeature.h>
18	19	#include <asm/hardirq.h>
19	20	#include <asm/intel-family.h>
	21	+#include <asm/intel_pt.h>
20	22	#include <asm/apic.h>
	23	+#include <asm/cpu_device_id.h>
21	24
22	25	#include "../perf_event.h"
23	26
..	..	@@ -238,11 +241,46 @@
238	241	EVENT_EXTRA_END
239	242	};
240	243
	244	+static struct event_constraint intel_icl_event_constraints[] = {
	245	+ FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
	246	+ FIXED_EVENT_CONSTRAINT(0x01c0, 0), /* INST_RETIRED.PREC_DIST */
	247	+ FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
	248	+ FIXED_EVENT_CONSTRAINT(0x0300, 2), /* CPU_CLK_UNHALTED.REF */
	249	+ FIXED_EVENT_CONSTRAINT(0x0400, 3), /* SLOTS */
	250	+ METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_RETIRING, 0),
	251	+ METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_BAD_SPEC, 1),
	252	+ METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_FE_BOUND, 2),
	253	+ METRIC_EVENT_CONSTRAINT(INTEL_TD_METRIC_BE_BOUND, 3),
	254	+ INTEL_EVENT_CONSTRAINT_RANGE(0x03, 0x0a, 0xf),
	255	+ INTEL_EVENT_CONSTRAINT_RANGE(0x1f, 0x28, 0xf),
	256	+ INTEL_EVENT_CONSTRAINT(0x32, 0xf), /* SW_PREFETCH_ACCESS.* */
	257	+ INTEL_EVENT_CONSTRAINT_RANGE(0x48, 0x56, 0xf),
	258	+ INTEL_EVENT_CONSTRAINT_RANGE(0x60, 0x8b, 0xf),
	259	+ INTEL_UEVENT_CONSTRAINT(0x04a3, 0xff), /* CYCLE_ACTIVITY.STALLS_TOTAL */
	260	+ INTEL_UEVENT_CONSTRAINT(0x10a3, 0xff), /* CYCLE_ACTIVITY.CYCLES_MEM_ANY */
	261	+ INTEL_UEVENT_CONSTRAINT(0x14a3, 0xff), /* CYCLE_ACTIVITY.STALLS_MEM_ANY */
	262	+ INTEL_EVENT_CONSTRAINT(0xa3, 0xf), /* CYCLE_ACTIVITY.* */
	263	+ INTEL_EVENT_CONSTRAINT_RANGE(0xa8, 0xb0, 0xf),
	264	+ INTEL_EVENT_CONSTRAINT_RANGE(0xb7, 0xbd, 0xf),
	265	+ INTEL_EVENT_CONSTRAINT_RANGE(0xd0, 0xe6, 0xf),
	266	+ INTEL_EVENT_CONSTRAINT(0xef, 0xf),
	267	+ INTEL_EVENT_CONSTRAINT_RANGE(0xf0, 0xf4, 0xf),
	268	+ EVENT_CONSTRAINT_END
	269	+};
	270	+
	271	+static struct extra_reg intel_icl_extra_regs[] __read_mostly = {
	272	+ INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x3fffffbfffull, RSP_0),
	273	+ INTEL_UEVENT_EXTRA_REG(0x01bb, MSR_OFFCORE_RSP_1, 0x3fffffbfffull, RSP_1),
	274	+ INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
	275	+ INTEL_UEVENT_EXTRA_REG(0x01c6, MSR_PEBS_FRONTEND, 0x7fff17, FE),
	276	+ EVENT_EXTRA_END
	277	+};
	278	+
241	279	EVENT_ATTR_STR(mem-loads, mem_ld_nhm, "event=0x0b,umask=0x10,ldlat=3");
242	280	EVENT_ATTR_STR(mem-loads, mem_ld_snb, "event=0xcd,umask=0x1,ldlat=3");
243	281	EVENT_ATTR_STR(mem-stores, mem_st_snb, "event=0xcd,umask=0x2");
244	282
245		-static struct attribute *nhm_events_attrs[] = {
	283	+static struct attribute *nhm_mem_events_attrs[] = {
246	284	EVENT_PTR(mem_ld_nhm),
247	285	NULL,
248	286	};
..	..	@@ -277,9 +315,13 @@
277	315	EVENT_ATTR_STR_HT(topdown-recovery-bubbles.scale, td_recovery_bubbles_scale,
278	316	"4", "2");
279	317
	318	+EVENT_ATTR_STR(slots, slots, "event=0x00,umask=0x4");
	319	+EVENT_ATTR_STR(topdown-retiring, td_retiring, "event=0x00,umask=0x80");
	320	+EVENT_ATTR_STR(topdown-bad-spec, td_bad_spec, "event=0x00,umask=0x81");
	321	+EVENT_ATTR_STR(topdown-fe-bound, td_fe_bound, "event=0x00,umask=0x82");
	322	+EVENT_ATTR_STR(topdown-be-bound, td_be_bound, "event=0x00,umask=0x83");
	323	+
280	324	static struct attribute *snb_events_attrs[] = {
281		- EVENT_PTR(mem_ld_snb),
282		- EVENT_PTR(mem_st_snb),
283	325	EVENT_PTR(td_slots_issued),
284	326	EVENT_PTR(td_slots_retired),
285	327	EVENT_PTR(td_fetch_bubbles),
..	..	@@ -287,6 +329,12 @@
287	329	EVENT_PTR(td_total_slots_scale),
288	330	EVENT_PTR(td_recovery_bubbles),
289	331	EVENT_PTR(td_recovery_bubbles_scale),
	332	+ NULL,
	333	+};
	334	+
	335	+static struct attribute *snb_mem_events_attrs[] = {
	336	+ EVENT_PTR(mem_ld_snb),
	337	+ EVENT_PTR(mem_st_snb),
290	338	NULL,
291	339	};
292	340
..	..	@@ -1822,6 +1870,45 @@
1822	1870	},
1823	1871	};
1824	1872
	1873	+#define TNT_LOCAL_DRAM BIT_ULL(26)
	1874	+#define TNT_DEMAND_READ GLM_DEMAND_DATA_RD
	1875	+#define TNT_DEMAND_WRITE GLM_DEMAND_RFO
	1876	+#define TNT_LLC_ACCESS GLM_ANY_RESPONSE
	1877	+#define TNT_SNP_ANY (SNB_SNP_NOT_NEEDED\|SNB_SNP_MISS\| \
	1878	+ SNB_NO_FWD\|SNB_SNP_FWD\|SNB_HITM)
	1879	+#define TNT_LLC_MISS (TNT_SNP_ANY\|SNB_NON_DRAM\|TNT_LOCAL_DRAM)
	1880	+
	1881	+static __initconst const u64 tnt_hw_cache_extra_regs
	1882	+ [PERF_COUNT_HW_CACHE_MAX]
	1883	+ [PERF_COUNT_HW_CACHE_OP_MAX]
	1884	+ [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
	1885	+ [C(LL)] = {
	1886	+ [C(OP_READ)] = {
	1887	+ [C(RESULT_ACCESS)] = TNT_DEMAND_READ\|
	1888	+ TNT_LLC_ACCESS,
	1889	+ [C(RESULT_MISS)] = TNT_DEMAND_READ\|
	1890	+ TNT_LLC_MISS,
	1891	+ },
	1892	+ [C(OP_WRITE)] = {
	1893	+ [C(RESULT_ACCESS)] = TNT_DEMAND_WRITE\|
	1894	+ TNT_LLC_ACCESS,
	1895	+ [C(RESULT_MISS)] = TNT_DEMAND_WRITE\|
	1896	+ TNT_LLC_MISS,
	1897	+ },
	1898	+ [C(OP_PREFETCH)] = {
	1899	+ [C(RESULT_ACCESS)] = 0x0,
	1900	+ [C(RESULT_MISS)] = 0x0,
	1901	+ },
	1902	+ },
	1903	+};
	1904	+
	1905	+static struct extra_reg intel_tnt_extra_regs[] __read_mostly = {
	1906	+ /* must define OFFCORE_RSP_X first, see intel_fixup_er() */
	1907	+ INTEL_UEVENT_EXTRA_REG(0x01b7, MSR_OFFCORE_RSP_0, 0x800ff0ffffff9fffull, RSP_0),
	1908	+ INTEL_UEVENT_EXTRA_REG(0x02b7, MSR_OFFCORE_RSP_1, 0xff0ffffff9fffull, RSP_1),
	1909	+ EVENT_EXTRA_END
	1910	+};
	1911	+
1825	1912	#define KNL_OT_L2_HITE BIT_ULL(19) /* Other Tile L2 Hit */
1826	1913	#define KNL_OT_L2_HITF BIT_ULL(20) /* Other Tile L2 Hit */
1827	1914	#define KNL_MCDRAM_LOCAL BIT_ULL(21)
..	..	@@ -1870,6 +1957,14 @@
1870	1957	* intel_bts events don't coexist with intel PMU's BTS events because of
1871	1958	* x86_add_exclusive(x86_lbr_exclusive_lbr); there's no need to keep them
1872	1959	* disabled around intel PMU's event batching etc, only inside the PMI handler.
	1960	+ *
	1961	+ * Avoid PEBS_ENABLE MSR access in PMIs.
	1962	+ * The GLOBAL_CTRL has been disabled. All the counters do not count anymore.
	1963	+ * It doesn't matter if the PEBS is enabled or not.
	1964	+ * Usually, the PEBS status are not changed in PMIs. It's unnecessary to
	1965	+ * access PEBS_ENABLE MSR in disable_all()/enable_all().
	1966	+ * However, there are some cases which may change PEBS status, e.g. PMI
	1967	+ * throttle. The PEBS_ENABLE should be updated where the status changes.
1873	1968	*/
1874	1969	static void __intel_pmu_disable_all(void)
1875	1970	{
..	..	@@ -1879,13 +1974,12 @@
1879	1974
1880	1975	if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask))
1881	1976	intel_pmu_disable_bts();
1882		-
1883		- intel_pmu_pebs_disable_all();
1884	1977	}
1885	1978
1886	1979	static void intel_pmu_disable_all(void)
1887	1980	{
1888	1981	__intel_pmu_disable_all();
	1982	+ intel_pmu_pebs_disable_all();
1889	1983	intel_pmu_lbr_disable_all();
1890	1984	}
1891	1985
..	..	@@ -1893,7 +1987,6 @@
1893	1987	{
1894	1988	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1895	1989
1896		- intel_pmu_pebs_enable_all();
1897	1990	intel_pmu_lbr_enable_all(pmi);
1898	1991	wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL,
1899	1992	x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask);
..	..	@@ -1911,6 +2004,7 @@
1911	2004
1912	2005	static void intel_pmu_enable_all(int added)
1913	2006	{
	2007	+ intel_pmu_pebs_enable_all();
1914	2008	__intel_pmu_enable_all(added, false);
1915	2009	}
1916	2010
..	..	@@ -1926,7 +2020,7 @@
1926	2020	* in sequence on the same PMC or on different PMCs.
1927	2021	*
1928	2022	* In practise it appears some of these events do in fact count, and
1929		- * we need to programm all 4 events.
	2023	+ * we need to program all 4 events.
1930	2024	*/
1931	2025	static void intel_pmu_nhm_workaround(void)
1932	2026	{
..	..	@@ -2010,7 +2104,7 @@
2010	2104	/*
2011	2105	* We're going to use PMC3, make sure TFA is set before we touch it.
2012	2106	*/
2013		- if (cntr == 3 && !cpuc->is_fake)
	2107	+ if (cntr == 3)
2014	2108	intel_set_tfa(cpuc, true);
2015	2109	}
2016	2110
..	..	@@ -2028,6 +2122,18 @@
2028	2122	intel_pmu_enable_all(added);
2029	2123	}
2030	2124
	2125	+static void enable_counter_freeze(void)
	2126	+{
	2127	+ update_debugctlmsr(get_debugctlmsr() \|
	2128	+ DEBUGCTLMSR_FREEZE_PERFMON_ON_PMI);
	2129	+}
	2130	+
	2131	+static void disable_counter_freeze(void)
	2132	+{
	2133	+ update_debugctlmsr(get_debugctlmsr() &
	2134	+ ~DEBUGCTLMSR_FREEZE_PERFMON_ON_PMI);
	2135	+}
	2136	+
2031	2137	static inline u64 intel_pmu_get_status(void)
2032	2138	{
2033	2139	u64 status;
..	..	@@ -2042,42 +2148,85 @@
2042	2148	wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
2043	2149	}
2044	2150
2045		-static void intel_pmu_disable_fixed(struct hw_perf_event *hwc)
	2151	+static inline bool event_is_checkpointed(struct perf_event *event)
2046	2152	{
2047		- int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
	2153	+ return unlikely(event->hw.config & HSW_IN_TX_CHECKPOINTED) != 0;
	2154	+}
	2155	+
	2156	+static inline void intel_set_masks(struct perf_event *event, int idx)
	2157	+{
	2158	+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
	2159	+
	2160	+ if (event->attr.exclude_host)
	2161	+ __set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_guest_mask);
	2162	+ if (event->attr.exclude_guest)
	2163	+ __set_bit(idx, (unsigned long *)&cpuc->intel_ctrl_host_mask);
	2164	+ if (event_is_checkpointed(event))
	2165	+ __set_bit(idx, (unsigned long *)&cpuc->intel_cp_status);
	2166	+}
	2167	+
	2168	+static inline void intel_clear_masks(struct perf_event *event, int idx)
	2169	+{
	2170	+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
	2171	+
	2172	+ __clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_guest_mask);
	2173	+ __clear_bit(idx, (unsigned long *)&cpuc->intel_ctrl_host_mask);
	2174	+ __clear_bit(idx, (unsigned long *)&cpuc->intel_cp_status);
	2175	+}
	2176	+
	2177	+static void intel_pmu_disable_fixed(struct perf_event *event)
	2178	+{
	2179	+ struct hw_perf_event *hwc = &event->hw;
2048	2180	u64 ctrl_val, mask;
	2181	+ int idx = hwc->idx;
2049	2182
2050		- mask = 0xfULL << (idx * 4);
	2183	+ if (is_topdown_idx(idx)) {
	2184	+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2051	2185
	2186	+ /*
	2187	+ * When there are other active TopDown events,
	2188	+ * don't disable the fixed counter 3.
	2189	+ */
	2190	+ if ((u64 )cpuc->active_mask & INTEL_PMC_OTHER_TOPDOWN_BITS(idx))
	2191	+ return;
	2192	+ idx = INTEL_PMC_IDX_FIXED_SLOTS;
	2193	+ }
	2194	+
	2195	+ intel_clear_masks(event, idx);
	2196	+
	2197	+ mask = 0xfULL << ((idx - INTEL_PMC_IDX_FIXED) * 4);
2052	2198	rdmsrl(hwc->config_base, ctrl_val);
2053	2199	ctrl_val &= ~mask;
2054	2200	wrmsrl(hwc->config_base, ctrl_val);
2055	2201	}
2056	2202
2057		-static inline bool event_is_checkpointed(struct perf_event *event)
2058		-{
2059		- return (event->hw.config & HSW_IN_TX_CHECKPOINTED) != 0;
2060		-}
2061		-
2062	2203	static void intel_pmu_disable_event(struct perf_event *event)
2063	2204	{
2064	2205	struct hw_perf_event *hwc = &event->hw;
2065		- struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
	2206	+ int idx = hwc->idx;
2066	2207
2067		- if (unlikely(hwc->idx == INTEL_PMC_IDX_FIXED_BTS)) {
	2208	+ switch (idx) {
	2209	+ case 0 ... INTEL_PMC_IDX_FIXED - 1:
	2210	+ intel_clear_masks(event, idx);
	2211	+ x86_pmu_disable_event(event);
	2212	+ break;
	2213	+ case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS - 1:
	2214	+ case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END:
	2215	+ intel_pmu_disable_fixed(event);
	2216	+ break;
	2217	+ case INTEL_PMC_IDX_FIXED_BTS:
2068	2218	intel_pmu_disable_bts();
2069	2219	intel_pmu_drain_bts_buffer();
2070	2220	return;
	2221	+ case INTEL_PMC_IDX_FIXED_VLBR:
	2222	+ intel_clear_masks(event, idx);
	2223	+ break;
	2224	+ default:
	2225	+ intel_clear_masks(event, idx);
	2226	+ pr_warn("Failed to disable the event with invalid index %d\n",
	2227	+ idx);
	2228	+ return;
2071	2229	}
2072		-
2073		- cpuc->intel_ctrl_guest_mask &= ~(1ull << hwc->idx);
2074		- cpuc->intel_ctrl_host_mask &= ~(1ull << hwc->idx);
2075		- cpuc->intel_cp_status &= ~(1ull << hwc->idx);
2076		-
2077		- if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL))
2078		- intel_pmu_disable_fixed(hwc);
2079		- else
2080		- x86_pmu_disable_event(event);
2081	2230
2082	2231	/*
2083	2232	* Needs to be called after x86_pmu_disable_event,
..	..	@@ -2095,10 +2244,189 @@
2095	2244	intel_pmu_pebs_del(event);
2096	2245	}
2097	2246
	2247	+static int icl_set_topdown_event_period(struct perf_event *event)
	2248	+{
	2249	+ struct hw_perf_event *hwc = &event->hw;
	2250	+ s64 left = local64_read(&hwc->period_left);
	2251	+
	2252	+ /*
	2253	+ * The values in PERF_METRICS MSR are derived from fixed counter 3.
	2254	+ * Software should start both registers, PERF_METRICS and fixed
	2255	+ * counter 3, from zero.
	2256	+ * Clear PERF_METRICS and Fixed counter 3 in initialization.
	2257	+ * After that, both MSRs will be cleared for each read.
	2258	+ * Don't need to clear them again.
	2259	+ */
	2260	+ if (left == x86_pmu.max_period) {
	2261	+ wrmsrl(MSR_CORE_PERF_FIXED_CTR3, 0);
	2262	+ wrmsrl(MSR_PERF_METRICS, 0);
	2263	+ hwc->saved_slots = 0;
	2264	+ hwc->saved_metric = 0;
	2265	+ }
	2266	+
	2267	+ if ((hwc->saved_slots) && is_slots_event(event)) {
	2268	+ wrmsrl(MSR_CORE_PERF_FIXED_CTR3, hwc->saved_slots);
	2269	+ wrmsrl(MSR_PERF_METRICS, hwc->saved_metric);
	2270	+ }
	2271	+
	2272	+ perf_event_update_userpage(event);
	2273	+
	2274	+ return 0;
	2275	+}
	2276	+
	2277	+static inline u64 icl_get_metrics_event_value(u64 metric, u64 slots, int idx)
	2278	+{
	2279	+ u32 val;
	2280	+
	2281	+ /*
	2282	+ * The metric is reported as an 8bit integer fraction
	2283	+ * suming up to 0xff.
	2284	+ * slots-in-metric = (Metric / 0xff) * slots
	2285	+ */
	2286	+ val = (metric >> ((idx - INTEL_PMC_IDX_METRIC_BASE) * 8)) & 0xff;
	2287	+ return mul_u64_u32_div(slots, val, 0xff);
	2288	+}
	2289	+
	2290	+static u64 icl_get_topdown_value(struct perf_event *event,
	2291	+ u64 slots, u64 metrics)
	2292	+{
	2293	+ int idx = event->hw.idx;
	2294	+ u64 delta;
	2295	+
	2296	+ if (is_metric_idx(idx))
	2297	+ delta = icl_get_metrics_event_value(metrics, slots, idx);
	2298	+ else
	2299	+ delta = slots;
	2300	+
	2301	+ return delta;
	2302	+}
	2303	+
	2304	+static void __icl_update_topdown_event(struct perf_event *event,
	2305	+ u64 slots, u64 metrics,
	2306	+ u64 last_slots, u64 last_metrics)
	2307	+{
	2308	+ u64 delta, last = 0;
	2309	+
	2310	+ delta = icl_get_topdown_value(event, slots, metrics);
	2311	+ if (last_slots)
	2312	+ last = icl_get_topdown_value(event, last_slots, last_metrics);
	2313	+
	2314	+ /*
	2315	+ * The 8bit integer fraction of metric may be not accurate,
	2316	+ * especially when the changes is very small.
	2317	+ * For example, if only a few bad_spec happens, the fraction
	2318	+ * may be reduced from 1 to 0. If so, the bad_spec event value
	2319	+ * will be 0 which is definitely less than the last value.
	2320	+ * Avoid update event->count for this case.
	2321	+ */
	2322	+ if (delta > last) {
	2323	+ delta -= last;
	2324	+ local64_add(delta, &event->count);
	2325	+ }
	2326	+}
	2327	+
	2328	+static void update_saved_topdown_regs(struct perf_event *event,
	2329	+ u64 slots, u64 metrics)
	2330	+{
	2331	+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
	2332	+ struct perf_event *other;
	2333	+ int idx;
	2334	+
	2335	+ event->hw.saved_slots = slots;
	2336	+ event->hw.saved_metric = metrics;
	2337	+
	2338	+ for_each_set_bit(idx, cpuc->active_mask, INTEL_PMC_IDX_TD_BE_BOUND + 1) {
	2339	+ if (!is_topdown_idx(idx))
	2340	+ continue;
	2341	+ other = cpuc->events[idx];
	2342	+ other->hw.saved_slots = slots;
	2343	+ other->hw.saved_metric = metrics;
	2344	+ }
	2345	+}
	2346	+
	2347	+/*
	2348	+ * Update all active Topdown events.
	2349	+ *
	2350	+ * The PERF_METRICS and Fixed counter 3 are read separately. The values may be
	2351	+ * modify by a NMI. PMU has to be disabled before calling this function.
	2352	+ */
	2353	+static u64 icl_update_topdown_event(struct perf_event *event)
	2354	+{
	2355	+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
	2356	+ struct perf_event *other;
	2357	+ u64 slots, metrics;
	2358	+ bool reset = true;
	2359	+ int idx;
	2360	+
	2361	+ /* read Fixed counter 3 */
	2362	+ rdpmcl((3 \| INTEL_PMC_FIXED_RDPMC_BASE), slots);
	2363	+ if (!slots)
	2364	+ return 0;
	2365	+
	2366	+ /* read PERF_METRICS */
	2367	+ rdpmcl(INTEL_PMC_FIXED_RDPMC_METRICS, metrics);
	2368	+
	2369	+ for_each_set_bit(idx, cpuc->active_mask, INTEL_PMC_IDX_TD_BE_BOUND + 1) {
	2370	+ if (!is_topdown_idx(idx))
	2371	+ continue;
	2372	+ other = cpuc->events[idx];
	2373	+ __icl_update_topdown_event(other, slots, metrics,
	2374	+ event ? event->hw.saved_slots : 0,
	2375	+ event ? event->hw.saved_metric : 0);
	2376	+ }
	2377	+
	2378	+ /*
	2379	+ * Check and update this event, which may have been cleared
	2380	+ * in active_mask e.g. x86_pmu_stop()
	2381	+ */
	2382	+ if (event && !test_bit(event->hw.idx, cpuc->active_mask)) {
	2383	+ __icl_update_topdown_event(event, slots, metrics,
	2384	+ event->hw.saved_slots,
	2385	+ event->hw.saved_metric);
	2386	+
	2387	+ /*
	2388	+ * In x86_pmu_stop(), the event is cleared in active_mask first,
	2389	+ * then drain the delta, which indicates context switch for
	2390	+ * counting.
	2391	+ * Save metric and slots for context switch.
	2392	+ * Don't need to reset the PERF_METRICS and Fixed counter 3.
	2393	+ * Because the values will be restored in next schedule in.
	2394	+ */
	2395	+ update_saved_topdown_regs(event, slots, metrics);
	2396	+ reset = false;
	2397	+ }
	2398	+
	2399	+ if (reset) {
	2400	+ /* The fixed counter 3 has to be written before the PERF_METRICS. */
	2401	+ wrmsrl(MSR_CORE_PERF_FIXED_CTR3, 0);
	2402	+ wrmsrl(MSR_PERF_METRICS, 0);
	2403	+ if (event)
	2404	+ update_saved_topdown_regs(event, 0, 0);
	2405	+ }
	2406	+
	2407	+ return slots;
	2408	+}
	2409	+
	2410	+static void intel_pmu_read_topdown_event(struct perf_event *event)
	2411	+{
	2412	+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
	2413	+
	2414	+ /* Only need to call update_topdown_event() once for group read. */
	2415	+ if ((cpuc->txn_flags & PERF_PMU_TXN_READ) &&
	2416	+ !is_slots_event(event))
	2417	+ return;
	2418	+
	2419	+ perf_pmu_disable(event->pmu);
	2420	+ x86_pmu.update_topdown_event(event);
	2421	+ perf_pmu_enable(event->pmu);
	2422	+}
	2423	+
2098	2424	static void intel_pmu_read_event(struct perf_event *event)
2099	2425	{
2100	2426	if (event->hw.flags & PERF_X86_EVENT_AUTO_RELOAD)
2101	2427	intel_pmu_auto_reload_read(event);
	2428	+ else if (is_topdown_count(event) && x86_pmu.update_topdown_event)
	2429	+ intel_pmu_read_topdown_event(event);
2102	2430	else
2103	2431	x86_perf_event_update(event);
2104	2432	}
..	..	@@ -2106,8 +2434,22 @@
2106	2434	static void intel_pmu_enable_fixed(struct perf_event *event)
2107	2435	{
2108	2436	struct hw_perf_event *hwc = &event->hw;
2109		- int idx = hwc->idx - INTEL_PMC_IDX_FIXED;
2110	2437	u64 ctrl_val, mask, bits = 0;
	2438	+ int idx = hwc->idx;
	2439	+
	2440	+ if (is_topdown_idx(idx)) {
	2441	+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
	2442	+ /*
	2443	+ * When there are other active TopDown events,
	2444	+ * don't enable the fixed counter 3 again.
	2445	+ */
	2446	+ if ((u64 )cpuc->active_mask & INTEL_PMC_OTHER_TOPDOWN_BITS(idx))
	2447	+ return;
	2448	+
	2449	+ idx = INTEL_PMC_IDX_FIXED_SLOTS;
	2450	+ }
	2451	+
	2452	+ intel_set_masks(event, idx);
2111	2453
2112	2454	/*
2113	2455	* Enable IRQ generation (0x8), if not PEBS,
..	..	@@ -2127,8 +2469,14 @@
2127	2469	if (x86_pmu.version > 2 && hwc->config & ARCH_PERFMON_EVENTSEL_ANY)
2128	2470	bits \|= 0x4;
2129	2471
	2472	+ idx -= INTEL_PMC_IDX_FIXED;
2130	2473	bits <<= (idx * 4);
2131	2474	mask = 0xfULL << (idx * 4);
	2475	+
	2476	+ if (x86_pmu.intel_cap.pebs_baseline && event->attr.precise_ip) {
	2477	+ bits \|= ICL_FIXED_0_ADAPTIVE << (idx * 4);
	2478	+ mask \|= ICL_FIXED_0_ADAPTIVE << (idx * 4);
	2479	+ }
2132	2480
2133	2481	rdmsrl(hwc->config_base, ctrl_val);
2134	2482	ctrl_val &= ~mask;
..	..	@@ -2139,33 +2487,32 @@
2139	2487	static void intel_pmu_enable_event(struct perf_event *event)
2140	2488	{
2141	2489	struct hw_perf_event *hwc = &event->hw;
2142		- struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
2143		-
2144		- if (unlikely(hwc->idx == INTEL_PMC_IDX_FIXED_BTS)) {
2145		- if (!__this_cpu_read(cpu_hw_events.enabled))
2146		- return;
2147		-
2148		- intel_pmu_enable_bts(hwc->config);
2149		- return;
2150		- }
2151		-
2152		- if (event->attr.exclude_host)
2153		- cpuc->intel_ctrl_guest_mask \|= (1ull << hwc->idx);
2154		- if (event->attr.exclude_guest)
2155		- cpuc->intel_ctrl_host_mask \|= (1ull << hwc->idx);
2156		-
2157		- if (unlikely(event_is_checkpointed(event)))
2158		- cpuc->intel_cp_status \|= (1ull << hwc->idx);
	2490	+ int idx = hwc->idx;
2159	2491
2160	2492	if (unlikely(event->attr.precise_ip))
2161	2493	intel_pmu_pebs_enable(event);
2162	2494
2163		- if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL)) {
	2495	+ switch (idx) {
	2496	+ case 0 ... INTEL_PMC_IDX_FIXED - 1:
	2497	+ intel_set_masks(event, idx);
	2498	+ __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
	2499	+ break;
	2500	+ case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS - 1:
	2501	+ case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END:
2164	2502	intel_pmu_enable_fixed(event);
2165		- return;
	2503	+ break;
	2504	+ case INTEL_PMC_IDX_FIXED_BTS:
	2505	+ if (!__this_cpu_read(cpu_hw_events.enabled))
	2506	+ return;
	2507	+ intel_pmu_enable_bts(hwc->config);
	2508	+ break;
	2509	+ case INTEL_PMC_IDX_FIXED_VLBR:
	2510	+ intel_set_masks(event, idx);
	2511	+ break;
	2512	+ default:
	2513	+ pr_warn("Failed to enable the event with invalid index %d\n",
	2514	+ idx);
2166	2515	}
2167		-
2168		- __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
2169	2516	}
2170	2517
2171	2518	static void intel_pmu_add_event(struct perf_event *event)
..	..	@@ -2235,15 +2582,221 @@
2235	2582	local_irq_restore(flags);
2236	2583	}
2237	2584
	2585	+static int handle_pmi_common(struct pt_regs *regs, u64 status)
	2586	+{
	2587	+ struct perf_sample_data data;
	2588	+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
	2589	+ struct perf_guest_info_callbacks *guest_cbs;
	2590	+ int bit;
	2591	+ int handled = 0;
	2592	+
	2593	+ inc_irq_stat(apic_perf_irqs);
	2594	+
	2595	+ /*
	2596	+ * Ignore a range of extra bits in status that do not indicate
	2597	+ * overflow by themselves.
	2598	+ */
	2599	+ status &= ~(GLOBAL_STATUS_COND_CHG \|
	2600	+ GLOBAL_STATUS_ASIF \|
	2601	+ GLOBAL_STATUS_LBRS_FROZEN);
	2602	+ if (!status)
	2603	+ return 0;
	2604	+ /*
	2605	+ * In case multiple PEBS events are sampled at the same time,
	2606	+ * it is possible to have GLOBAL_STATUS bit 62 set indicating
	2607	+ * PEBS buffer overflow and also seeing at most 3 PEBS counters
	2608	+ * having their bits set in the status register. This is a sign
	2609	+ * that there was at least one PEBS record pending at the time
	2610	+ * of the PMU interrupt. PEBS counters must only be processed
	2611	+ * via the drain_pebs() calls and not via the regular sample
	2612	+ * processing loop coming after that the function, otherwise
	2613	+ * phony regular samples may be generated in the sampling buffer
	2614	+ * not marked with the EXACT tag. Another possibility is to have
	2615	+ * one PEBS event and at least one non-PEBS event whic hoverflows
	2616	+ * while PEBS has armed. In this case, bit 62 of GLOBAL_STATUS will
	2617	+ * not be set, yet the overflow status bit for the PEBS counter will
	2618	+ * be on Skylake.
	2619	+ *
	2620	+ * To avoid this problem, we systematically ignore the PEBS-enabled
	2621	+ * counters from the GLOBAL_STATUS mask and we always process PEBS
	2622	+ * events via drain_pebs().
	2623	+ */
	2624	+ if (x86_pmu.flags & PMU_FL_PEBS_ALL)
	2625	+ status &= ~cpuc->pebs_enabled;
	2626	+ else
	2627	+ status &= ~(cpuc->pebs_enabled & PEBS_COUNTER_MASK);
	2628	+
	2629	+ /*
	2630	+ * PEBS overflow sets bit 62 in the global status register
	2631	+ */
	2632	+ if (__test_and_clear_bit(GLOBAL_STATUS_BUFFER_OVF_BIT, (unsigned long *)&status)) {
	2633	+ u64 pebs_enabled = cpuc->pebs_enabled;
	2634	+
	2635	+ handled++;
	2636	+ x86_pmu.drain_pebs(regs, &data);
	2637	+ status &= x86_pmu.intel_ctrl \| GLOBAL_STATUS_TRACE_TOPAPMI;
	2638	+
	2639	+ /*
	2640	+ * PMI throttle may be triggered, which stops the PEBS event.
	2641	+ * Although cpuc->pebs_enabled is updated accordingly, the
	2642	+ * MSR_IA32_PEBS_ENABLE is not updated. Because the
	2643	+ * cpuc->enabled has been forced to 0 in PMI.
	2644	+ * Update the MSR if pebs_enabled is changed.
	2645	+ */
	2646	+ if (pebs_enabled != cpuc->pebs_enabled)
	2647	+ wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
	2648	+ }
	2649	+
	2650	+ /*
	2651	+ * Intel PT
	2652	+ */
	2653	+ if (__test_and_clear_bit(GLOBAL_STATUS_TRACE_TOPAPMI_BIT, (unsigned long *)&status)) {
	2654	+ handled++;
	2655	+
	2656	+ guest_cbs = perf_get_guest_cbs();
	2657	+ if (unlikely(guest_cbs && guest_cbs->is_in_guest() &&
	2658	+ guest_cbs->handle_intel_pt_intr))
	2659	+ guest_cbs->handle_intel_pt_intr();
	2660	+ else
	2661	+ intel_pt_interrupt();
	2662	+ }
	2663	+
	2664	+ /*
	2665	+ * Intel Perf mertrics
	2666	+ */
	2667	+ if (__test_and_clear_bit(GLOBAL_STATUS_PERF_METRICS_OVF_BIT, (unsigned long *)&status)) {
	2668	+ handled++;
	2669	+ if (x86_pmu.update_topdown_event)
	2670	+ x86_pmu.update_topdown_event(NULL);
	2671	+ }
	2672	+
	2673	+ /*
	2674	+ * Checkpointed counters can lead to 'spurious' PMIs because the
	2675	+ * rollback caused by the PMI will have cleared the overflow status
	2676	+ * bit. Therefore always force probe these counters.
	2677	+ */
	2678	+ status \|= cpuc->intel_cp_status;
	2679	+
	2680	+ for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
	2681	+ struct perf_event *event = cpuc->events[bit];
	2682	+
	2683	+ handled++;
	2684	+
	2685	+ if (!test_bit(bit, cpuc->active_mask))
	2686	+ continue;
	2687	+
	2688	+ if (!intel_pmu_save_and_restart(event))
	2689	+ continue;
	2690	+
	2691	+ perf_sample_data_init(&data, 0, event->hw.last_period);
	2692	+
	2693	+ if (has_branch_stack(event))
	2694	+ data.br_stack = &cpuc->lbr_stack;
	2695	+
	2696	+ if (perf_event_overflow(event, &data, regs))
	2697	+ x86_pmu_stop(event, 0);
	2698	+ }
	2699	+
	2700	+ return handled;
	2701	+}
	2702	+
	2703	+static bool disable_counter_freezing = true;
	2704	+static int __init intel_perf_counter_freezing_setup(char *s)
	2705	+{
	2706	+ bool res;
	2707	+
	2708	+ if (kstrtobool(s, &res))
	2709	+ return -EINVAL;
	2710	+
	2711	+ disable_counter_freezing = !res;
	2712	+ return 1;
	2713	+}
	2714	+__setup("perf_v4_pmi=", intel_perf_counter_freezing_setup);
	2715	+
	2716	+/*
	2717	+ * Simplified handler for Arch Perfmon v4:
	2718	+ * - We rely on counter freezing/unfreezing to enable/disable the PMU.
	2719	+ * This is done automatically on PMU ack.
	2720	+ * - Ack the PMU only after the APIC.
	2721	+ */
	2722	+
	2723	+static int intel_pmu_handle_irq_v4(struct pt_regs *regs)
	2724	+{
	2725	+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
	2726	+ int handled = 0;
	2727	+ bool bts = false;
	2728	+ u64 status;
	2729	+ int pmu_enabled = cpuc->enabled;
	2730	+ int loops = 0;
	2731	+
	2732	+ /* PMU has been disabled because of counter freezing */
	2733	+ cpuc->enabled = 0;
	2734	+ if (test_bit(INTEL_PMC_IDX_FIXED_BTS, cpuc->active_mask)) {
	2735	+ bts = true;
	2736	+ intel_bts_disable_local();
	2737	+ handled = intel_pmu_drain_bts_buffer();
	2738	+ handled += intel_bts_interrupt();
	2739	+ }
	2740	+ status = intel_pmu_get_status();
	2741	+ if (!status)
	2742	+ goto done;
	2743	+again:
	2744	+ intel_pmu_lbr_read();
	2745	+ if (++loops > 100) {
	2746	+ static bool warned;
	2747	+
	2748	+ if (!warned) {
	2749	+ WARN(1, "perfevents: irq loop stuck!\n");
	2750	+ perf_event_print_debug();
	2751	+ warned = true;
	2752	+ }
	2753	+ intel_pmu_reset();
	2754	+ goto done;
	2755	+ }
	2756	+
	2757	+
	2758	+ handled += handle_pmi_common(regs, status);
	2759	+done:
	2760	+ /* Ack the PMI in the APIC */
	2761	+ apic_write(APIC_LVTPC, APIC_DM_NMI);
	2762	+
	2763	+ /*
	2764	+ * The counters start counting immediately while ack the status.
	2765	+ * Make it as close as possible to IRET. This avoids bogus
	2766	+ * freezing on Skylake CPUs.
	2767	+ */
	2768	+ if (status) {
	2769	+ intel_pmu_ack_status(status);
	2770	+ } else {
	2771	+ /*
	2772	+ * CPU may issues two PMIs very close to each other.
	2773	+ * When the PMI handler services the first one, the
	2774	+ * GLOBAL_STATUS is already updated to reflect both.
	2775	+ * When it IRETs, the second PMI is immediately
	2776	+ * handled and it sees clear status. At the meantime,
	2777	+ * there may be a third PMI, because the freezing bit
	2778	+ * isn't set since the ack in first PMI handlers.
	2779	+ * Double check if there is more work to be done.
	2780	+ */
	2781	+ status = intel_pmu_get_status();
	2782	+ if (status)
	2783	+ goto again;
	2784	+ }
	2785	+
	2786	+ if (bts)
	2787	+ intel_bts_enable_local();
	2788	+ cpuc->enabled = pmu_enabled;
	2789	+ return handled;
	2790	+}
	2791	+
2238	2792	/*
2239	2793	* This handler is triggered by the local APIC, so the APIC IRQ handling
2240	2794	* rules apply:
2241	2795	*/
2242	2796	static int intel_pmu_handle_irq(struct pt_regs *regs)
2243	2797	{
2244		- struct perf_sample_data data;
2245	2798	struct cpu_hw_events *cpuc;
2246		- int bit, loops;
	2799	+ int loops;
2247	2800	u64 status;
2248	2801	int handled;
2249	2802	int pmu_enabled;
..	..	@@ -2275,7 +2828,8 @@
2275	2828	intel_pmu_lbr_read();
2276	2829	intel_pmu_ack_status(status);
2277	2830	if (++loops > 100) {
2278		- static bool warned = false;
	2831	+ static bool warned;
	2832	+
2279	2833	if (!warned) {
2280	2834	WARN(1, "perfevents: irq loop stuck!\n");
2281	2835	perf_event_print_debug();
..	..	@@ -2285,86 +2839,7 @@
2285	2839	goto done;
2286	2840	}
2287	2841
2288		- inc_irq_stat(apic_perf_irqs);
2289		-
2290		-
2291		- /*
2292		- * Ignore a range of extra bits in status that do not indicate
2293		- * overflow by themselves.
2294		- */
2295		- status &= ~(GLOBAL_STATUS_COND_CHG \|
2296		- GLOBAL_STATUS_ASIF \|
2297		- GLOBAL_STATUS_LBRS_FROZEN);
2298		- if (!status)
2299		- goto done;
2300		- /*
2301		- * In case multiple PEBS events are sampled at the same time,
2302		- * it is possible to have GLOBAL_STATUS bit 62 set indicating
2303		- * PEBS buffer overflow and also seeing at most 3 PEBS counters
2304		- * having their bits set in the status register. This is a sign
2305		- * that there was at least one PEBS record pending at the time
2306		- * of the PMU interrupt. PEBS counters must only be processed
2307		- * via the drain_pebs() calls and not via the regular sample
2308		- * processing loop coming after that the function, otherwise
2309		- * phony regular samples may be generated in the sampling buffer
2310		- * not marked with the EXACT tag. Another possibility is to have
2311		- * one PEBS event and at least one non-PEBS event whic hoverflows
2312		- * while PEBS has armed. In this case, bit 62 of GLOBAL_STATUS will
2313		- * not be set, yet the overflow status bit for the PEBS counter will
2314		- * be on Skylake.
2315		- *
2316		- * To avoid this problem, we systematically ignore the PEBS-enabled
2317		- * counters from the GLOBAL_STATUS mask and we always process PEBS
2318		- * events via drain_pebs().
2319		- */
2320		- if (x86_pmu.flags & PMU_FL_PEBS_ALL)
2321		- status &= ~cpuc->pebs_enabled;
2322		- else
2323		- status &= ~(cpuc->pebs_enabled & PEBS_COUNTER_MASK);
2324		-
2325		- /*
2326		- * PEBS overflow sets bit 62 in the global status register
2327		- */
2328		- if (__test_and_clear_bit(62, (unsigned long *)&status)) {
2329		- handled++;
2330		- x86_pmu.drain_pebs(regs);
2331		- status &= x86_pmu.intel_ctrl \| GLOBAL_STATUS_TRACE_TOPAPMI;
2332		- }
2333		-
2334		- /*
2335		- * Intel PT
2336		- */
2337		- if (__test_and_clear_bit(55, (unsigned long *)&status)) {
2338		- handled++;
2339		- intel_pt_interrupt();
2340		- }
2341		-
2342		- /*
2343		- * Checkpointed counters can lead to 'spurious' PMIs because the
2344		- * rollback caused by the PMI will have cleared the overflow status
2345		- * bit. Therefore always force probe these counters.
2346		- */
2347		- status \|= cpuc->intel_cp_status;
2348		-
2349		- for_each_set_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
2350		- struct perf_event *event = cpuc->events[bit];
2351		-
2352		- handled++;
2353		-
2354		- if (!test_bit(bit, cpuc->active_mask))
2355		- continue;
2356		-
2357		- if (!intel_pmu_save_and_restart(event))
2358		- continue;
2359		-
2360		- perf_sample_data_init(&data, 0, event->hw.last_period);
2361		-
2362		- if (has_branch_stack(event))
2363		- data.br_stack = &cpuc->lbr_stack;
2364		-
2365		- if (perf_event_overflow(event, &data, regs))
2366		- x86_pmu_stop(event, 0);
2367		- }
	2842	+ handled += handle_pmi_common(regs, status);
2368	2843
2369	2844	/*
2370	2845	* Repeat if there is more work to be done:
..	..	@@ -2395,6 +2870,22 @@
2395	2870	{
2396	2871	if (unlikely(intel_pmu_has_bts(event)))
2397	2872	return &bts_constraint;
	2873	+
	2874	+ return NULL;
	2875	+}
	2876	+
	2877	+/*
	2878	+ * Note: matches a fake event, like Fixed2.
	2879	+ */
	2880	+static struct event_constraint *
	2881	+intel_vlbr_constraints(struct perf_event *event)
	2882	+{
	2883	+ struct event_constraint *c = &vlbr_constraint;
	2884	+
	2885	+ if (unlikely(constraint_match(c, event->hw.config))) {
	2886	+ event->hw.flags \|= c->flags;
	2887	+ return c;
	2888	+ }
2398	2889
2399	2890	return NULL;
2400	2891	}
..	..	@@ -2573,7 +3064,7 @@
2573	3064
2574	3065	if (x86_pmu.event_constraints) {
2575	3066	for_each_event_constraint(c, x86_pmu.event_constraints) {
2576		- if ((event->hw.config & c->cmask) == c->code) {
	3067	+ if (constraint_match(c, event->hw.config)) {
2577	3068	event->hw.flags \|= c->flags;
2578	3069	return c;
2579	3070	}
..	..	@@ -2588,6 +3079,10 @@
2588	3079	struct perf_event *event)
2589	3080	{
2590	3081	struct event_constraint *c;
	3082	+
	3083	+ c = intel_vlbr_constraints(event);
	3084	+ if (c)
	3085	+ return c;
2591	3086
2592	3087	c = intel_bts_constraints(event);
2593	3088	if (c)
..	..	@@ -2723,7 +3218,7 @@
2723	3218	struct intel_excl_cntrs *excl_cntrs = cpuc->excl_cntrs;
2724	3219	struct intel_excl_states *xlo;
2725	3220	int tid = cpuc->excl_thread_id;
2726		- int is_excl, i;
	3221	+ int is_excl, i, w;
2727	3222
2728	3223	/*
2729	3224	* validating a group does not require
..	..	@@ -2779,35 +3274,39 @@
2779	3274	* SHARED : sibling counter measuring non-exclusive event
2780	3275	* UNUSED : sibling counter unused
2781	3276	*/
	3277	+ w = c->weight;
2782	3278	for_each_set_bit(i, c->idxmsk, X86_PMC_IDX_MAX) {
2783	3279	/*
2784	3280	* exclusive event in sibling counter
2785	3281	* our corresponding counter cannot be used
2786	3282	* regardless of our event
2787	3283	*/
2788		- if (xlo->state[i] == INTEL_EXCL_EXCLUSIVE)
	3284	+ if (xlo->state[i] == INTEL_EXCL_EXCLUSIVE) {
2789	3285	__clear_bit(i, c->idxmsk);
	3286	+ w--;
	3287	+ continue;
	3288	+ }
2790	3289	/*
2791	3290	* if measuring an exclusive event, sibling
2792	3291	* measuring non-exclusive, then counter cannot
2793	3292	* be used
2794	3293	*/
2795		- if (is_excl && xlo->state[i] == INTEL_EXCL_SHARED)
	3294	+ if (is_excl && xlo->state[i] == INTEL_EXCL_SHARED) {
2796	3295	__clear_bit(i, c->idxmsk);
	3296	+ w--;
	3297	+ continue;
	3298	+ }
2797	3299	}
2798		-
2799		- /*
2800		- * recompute actual bit weight for scheduling algorithm
2801		- */
2802		- c->weight = hweight64(c->idxmsk64);
2803	3300
2804	3301	/*
2805	3302	* if we return an empty mask, then switch
2806	3303	* back to static empty constraint to avoid
2807	3304	* the cost of freeing later on
2808	3305	*/
2809		- if (c->weight == 0)
	3306	+ if (!w)
2810	3307	c = &emptyconstraint;
	3308	+
	3309	+ c->weight = w;
2811	3310
2812	3311	return c;
2813	3312	}
..	..	@@ -2816,11 +3315,9 @@
2816	3315	intel_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
2817	3316	struct perf_event *event)
2818	3317	{
2819		- struct event_constraint *c1 = NULL;
2820		- struct event_constraint *c2;
	3318	+ struct event_constraint c1, c2;
2821	3319
2822		- if (idx >= 0) /* fake does < 0 */
2823		- c1 = cpuc->event_constraint[idx];
	3320	+ c1 = cpuc->event_constraint[idx];
2824	3321
2825	3322	/*
2826	3323	* first time only
..	..	@@ -2828,7 +3325,8 @@
2828	3325	* - dynamic constraint: handled by intel_get_excl_constraints()
2829	3326	*/
2830	3327	c2 = __intel_get_event_constraints(cpuc, idx, event);
2831		- if (c1 && (c1->flags & PERF_X86_EVENT_DYNAMIC)) {
	3328	+ if (c1) {
	3329	+ WARN_ON_ONCE(!(c1->flags & PERF_X86_EVENT_DYNAMIC));
2832	3330	bitmap_copy(c1->idxmsk, c2->idxmsk, X86_PMC_IDX_MAX);
2833	3331	c1->weight = c2->weight;
2834	3332	c2 = c1;
..	..	@@ -3070,11 +3568,16 @@
3070	3568	return ret;
3071	3569
3072	3570	if (event->attr.precise_ip) {
	3571	+ if ((event->attr.config & INTEL_ARCH_EVENT_MASK) == INTEL_FIXED_VLBR_EVENT)
	3572	+ return -EINVAL;
	3573	+
3073	3574	if (!(event->attr.freq \|\| (event->attr.wakeup_events && !event->attr.watermark))) {
3074	3575	event->hw.flags \|= PERF_X86_EVENT_AUTO_RELOAD;
3075	3576	if (!(event->attr.sample_type &
3076		- ~intel_pmu_large_pebs_flags(event)))
	3577	+ ~intel_pmu_large_pebs_flags(event))) {
3077	3578	event->hw.flags \|= PERF_X86_EVENT_LARGE_PEBS;
	3579	+ event->attach_state \|= PERF_ATTACH_SCHED_CB;
	3580	+ }
3078	3581	}
3079	3582	if (x86_pmu.pebs_aliases)
3080	3583	x86_pmu.pebs_aliases(event);
..	..	@@ -3087,6 +3590,7 @@
3087	3590	ret = intel_pmu_setup_lbr_filter(event);
3088	3591	if (ret)
3089	3592	return ret;
	3593	+ event->attach_state \|= PERF_ATTACH_SCHED_CB;
3090	3594
3091	3595	/*
3092	3596	* BTS is set up earlier in this path, so don't account twice
..	..	@@ -3100,8 +3604,65 @@
3100	3604	}
3101	3605	}
3102	3606
	3607	+ if (event->attr.aux_output) {
	3608	+ if (!event->attr.precise_ip)
	3609	+ return -EINVAL;
	3610	+
	3611	+ event->hw.flags \|= PERF_X86_EVENT_PEBS_VIA_PT;
	3612	+ }
	3613	+
3103	3614	if (event->attr.type != PERF_TYPE_RAW)
3104	3615	return 0;
	3616	+
	3617	+ /*
	3618	+ * Config Topdown slots and metric events
	3619	+ *
	3620	+ * The slots event on Fixed Counter 3 can support sampling,
	3621	+ * which will be handled normally in x86_perf_event_update().
	3622	+ *
	3623	+ * Metric events don't support sampling and require being paired
	3624	+ * with a slots event as group leader. When the slots event
	3625	+ * is used in a metrics group, it too cannot support sampling.
	3626	+ */
	3627	+ if (x86_pmu.intel_cap.perf_metrics && is_topdown_event(event)) {
	3628	+ if (event->attr.config1 \|\| event->attr.config2)
	3629	+ return -EINVAL;
	3630	+
	3631	+ /*
	3632	+ * The TopDown metrics events and slots event don't
	3633	+ * support any filters.
	3634	+ */
	3635	+ if (event->attr.config & X86_ALL_EVENT_FLAGS)
	3636	+ return -EINVAL;
	3637	+
	3638	+ if (is_metric_event(event)) {
	3639	+ struct perf_event *leader = event->group_leader;
	3640	+
	3641	+ /* The metric events don't support sampling. */
	3642	+ if (is_sampling_event(event))
	3643	+ return -EINVAL;
	3644	+
	3645	+ /* The metric events require a slots group leader. */
	3646	+ if (!is_slots_event(leader))
	3647	+ return -EINVAL;
	3648	+
	3649	+ /*
	3650	+ * The leader/SLOTS must not be a sampling event for
	3651	+ * metric use; hardware requires it starts at 0 when used
	3652	+ * in conjunction with MSR_PERF_METRICS.
	3653	+ */
	3654	+ if (is_sampling_event(leader))
	3655	+ return -EINVAL;
	3656	+
	3657	+ event->event_caps \|= PERF_EV_CAP_SIBLING;
	3658	+ /*
	3659	+ * Only once we have a METRICs sibling do we
	3660	+ * need TopDown magic.
	3661	+ */
	3662	+ leader->hw.flags \|= PERF_X86_EVENT_TOPDOWN;
	3663	+ event->hw.flags \|= PERF_X86_EVENT_TOPDOWN;
	3664	+ }
	3665	+ }
3105	3666
3106	3667	if (!(event->attr.config & ARCH_PERFMON_EVENTSEL_ANY))
3107	3668	return 0;
..	..	@@ -3118,8 +3679,19 @@
3118	3679	return 0;
3119	3680	}
3120	3681
	3682	+#ifdef CONFIG_RETPOLINE
	3683	+static struct perf_guest_switch_msr core_guest_get_msrs(int nr);
	3684	+static struct perf_guest_switch_msr intel_guest_get_msrs(int nr);
	3685	+#endif
	3686	+
3121	3687	struct perf_guest_switch_msr perf_guest_get_msrs(int nr)
3122	3688	{
	3689	+#ifdef CONFIG_RETPOLINE
	3690	+ if (x86_pmu.guest_get_msrs == intel_guest_get_msrs)
	3691	+ return intel_guest_get_msrs(nr);
	3692	+ else if (x86_pmu.guest_get_msrs == core_guest_get_msrs)
	3693	+ return core_guest_get_msrs(nr);
	3694	+#endif
3123	3695	if (x86_pmu.guest_get_msrs)
3124	3696	return x86_pmu.guest_get_msrs(nr);
3125	3697	*nr = 0;
..	..	@@ -3135,16 +3707,27 @@
3135	3707	arr[0].msr = MSR_CORE_PERF_GLOBAL_CTRL;
3136	3708	arr[0].host = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_guest_mask;
3137	3709	arr[0].guest = x86_pmu.intel_ctrl & ~cpuc->intel_ctrl_host_mask;
3138		- /*
3139		- * If PMU counter has PEBS enabled it is not enough to disable counter
3140		- * on a guest entry since PEBS memory write can overshoot guest entry
3141		- * and corrupt guest memory. Disabling PEBS solves the problem.
3142		- */
3143		- arr[1].msr = MSR_IA32_PEBS_ENABLE;
3144		- arr[1].host = cpuc->pebs_enabled;
3145		- arr[1].guest = 0;
	3710	+ if (x86_pmu.flags & PMU_FL_PEBS_ALL)
	3711	+ arr[0].guest &= ~cpuc->pebs_enabled;
	3712	+ else
	3713	+ arr[0].guest &= ~(cpuc->pebs_enabled & PEBS_COUNTER_MASK);
	3714	+ *nr = 1;
3146	3715
3147		- *nr = 2;
	3716	+ if (x86_pmu.pebs && x86_pmu.pebs_no_isolation) {
	3717	+ /*
	3718	+ * If PMU counter has PEBS enabled it is not enough to
	3719	+ * disable counter on a guest entry since PEBS memory
	3720	+ * write can overshoot guest entry and corrupt guest
	3721	+ * memory. Disabling PEBS solves the problem.
	3722	+ *
	3723	+ * Don't do this if the CPU already enforces it.
	3724	+ */
	3725	+ arr[1].msr = MSR_IA32_PEBS_ENABLE;
	3726	+ arr[1].host = cpuc->pebs_enabled;
	3727	+ arr[1].guest = 0;
	3728	+ *nr = 2;
	3729	+ }
	3730	+
3148	3731	return arr;
3149	3732	}
3150	3733
..	..	@@ -3241,6 +3824,12 @@
3241	3824	static struct event_constraint counter2_constraint =
3242	3825	EVENT_CONSTRAINT(0, 0x4, 0);
3243	3826
	3827	+static struct event_constraint fixed0_constraint =
	3828	+ FIXED_EVENT_CONSTRAINT(0x00c0, 0);
	3829	+
	3830	+static struct event_constraint fixed0_counter0_constraint =
	3831	+ INTEL_ALL_EVENT_CONSTRAINT(0, 0x100000001ULL);
	3832	+
3244	3833	static struct event_constraint *
3245	3834	hsw_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
3246	3835	struct perf_event *event)
..	..	@@ -3260,6 +3849,21 @@
3260	3849	}
3261	3850
3262	3851	static struct event_constraint *
	3852	+icl_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
	3853	+ struct perf_event *event)
	3854	+{
	3855	+ /*
	3856	+ * Fixed counter 0 has less skid.
	3857	+ * Force instruction:ppp in Fixed counter 0
	3858	+ */
	3859	+ if ((event->attr.precise_ip == 3) &&
	3860	+ constraint_match(&fixed0_constraint, event->hw.config))
	3861	+ return &fixed0_constraint;
	3862	+
	3863	+ return hsw_get_event_constraints(cpuc, idx, event);
	3864	+}
	3865	+
	3866	+static struct event_constraint *
3263	3867	glp_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
3264	3868	struct perf_event *event)
3265	3869	{
..	..	@@ -3268,6 +3872,29 @@
3268	3872	/* :ppp means to do reduced skid PEBS which is PMC0 only. */
3269	3873	if (event->attr.precise_ip == 3)
3270	3874	return &counter0_constraint;
	3875	+
	3876	+ c = intel_get_event_constraints(cpuc, idx, event);
	3877	+
	3878	+ return c;
	3879	+}
	3880	+
	3881	+static struct event_constraint *
	3882	+tnt_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
	3883	+ struct perf_event *event)
	3884	+{
	3885	+ struct event_constraint *c;
	3886	+
	3887	+ /*
	3888	+ * :ppp means to do reduced skid PEBS,
	3889	+ * which is available on PMC0 and fixed counter 0.
	3890	+ */
	3891	+ if (event->attr.precise_ip == 3) {
	3892	+ /* Force instruction:ppp on PMC0 and Fixed counter 0 */
	3893	+ if (constraint_match(&fixed0_constraint, event->hw.config))
	3894	+ return &fixed0_counter0_constraint;
	3895	+
	3896	+ return &counter0_constraint;
	3897	+ }
3271	3898
3272	3899	c = intel_get_event_constraints(cpuc, idx, event);
3273	3900
..	..	@@ -3285,7 +3912,7 @@
3285	3912	/*
3286	3913	* Without TFA we must not use PMC3.
3287	3914	*/
3288		- if (!allow_tsx_force_abort && test_bit(3, c->idxmsk) && idx >= 0) {
	3915	+ if (!allow_tsx_force_abort && test_bit(3, c->idxmsk)) {
3289	3916	c = dyn_constraint(cpuc, c, idx);
3290	3917	c->idxmsk64 &= ~(1ULL << 3);
3291	3918	c->weight--;
..	..	@@ -3387,6 +4014,8 @@
3387	4014
3388	4015	int intel_cpuc_prepare(struct cpu_hw_events *cpuc, int cpu)
3389	4016	{
	4017	+ cpuc->pebs_record_size = x86_pmu.pebs_record_size;
	4018	+
3390	4019	if (x86_pmu.extra_regs \|\| x86_pmu.lbr_sel_map) {
3391	4020	cpuc->shared_regs = allocate_shared_regs(cpu);
3392	4021	if (!cpuc->shared_regs)
..	..	@@ -3464,6 +4093,20 @@
3464	4093	if (x86_pmu.version > 1)
3465	4094	flip_smm_bit(&x86_pmu.attr_freeze_on_smi);
3466	4095
	4096	+ if (x86_pmu.counter_freezing)
	4097	+ enable_counter_freeze();
	4098	+
	4099	+ /* Disable perf metrics if any added CPU doesn't support it. */
	4100	+ if (x86_pmu.intel_cap.perf_metrics) {
	4101	+ union perf_capabilities perf_cap;
	4102	+
	4103	+ rdmsrl(MSR_IA32_PERF_CAPABILITIES, perf_cap.capabilities);
	4104	+ if (!perf_cap.perf_metrics) {
	4105	+ x86_pmu.intel_cap.perf_metrics = 0;
	4106	+ x86_pmu.intel_ctrl &= ~(1ULL << GLOBAL_CTRL_EN_PERF_METRICS);
	4107	+ }
	4108	+ }
	4109	+
3467	4110	if (!cpuc->shared_regs)
3468	4111	return;
3469	4112
..	..	@@ -3523,6 +4166,9 @@
3523	4166	static void intel_pmu_cpu_dying(int cpu)
3524	4167	{
3525	4168	fini_debug_store_on_cpu(cpu);
	4169	+
	4170	+ if (x86_pmu.counter_freezing)
	4171	+ disable_counter_freeze();
3526	4172	}
3527	4173
3528	4174	void intel_cpuc_finish(struct cpu_hw_events *cpuc)
..	..	@@ -3551,9 +4197,23 @@
3551	4197	intel_pmu_lbr_sched_task(ctx, sched_in);
3552	4198	}
3553	4199
	4200	+static void intel_pmu_swap_task_ctx(struct perf_event_context *prev,
	4201	+ struct perf_event_context *next)
	4202	+{
	4203	+ intel_pmu_lbr_swap_task_ctx(prev, next);
	4204	+}
	4205	+
3554	4206	static int intel_pmu_check_period(struct perf_event *event, u64 value)
3555	4207	{
3556	4208	return intel_pmu_has_bts_period(event, value) ? -EINVAL : 0;
	4209	+}
	4210	+
	4211	+static int intel_pmu_aux_output_match(struct perf_event *event)
	4212	+{
	4213	+ if (!x86_pmu.intel_cap.pebs_output_pt_available)
	4214	+ return 0;
	4215	+
	4216	+ return is_intel_pt_event(event);
3557	4217	}
3558	4218
3559	4219	PMU_FORMAT_ATTR(offcore_rsp, "config1:0-63");
..	..	@@ -3638,9 +4298,12 @@
3638	4298	.cpu_dead = intel_pmu_cpu_dead,
3639	4299
3640	4300	.check_period = intel_pmu_check_period,
3641		-};
3642	4301
3643		-static struct attribute *intel_pmu_attrs[];
	4302	+ .lbr_reset = intel_pmu_lbr_reset_64,
	4303	+ .lbr_read = intel_pmu_lbr_read_64,
	4304	+ .lbr_save = intel_pmu_lbr_save,
	4305	+ .lbr_restore = intel_pmu_lbr_restore,
	4306	+};
3644	4307
3645	4308	static __initconst const struct x86_pmu intel_pmu = {
3646	4309	.name = "Intel",
..	..	@@ -3673,8 +4336,6 @@
3673	4336	.format_attrs = intel_arch3_formats_attr,
3674	4337	.events_sysfs_show = intel_event_sysfs_show,
3675	4338
3676		- .attrs = intel_pmu_attrs,
3677		-
3678	4339	.cpu_prepare = intel_pmu_cpu_prepare,
3679	4340	.cpu_starting = intel_pmu_cpu_starting,
3680	4341	.cpu_dying = intel_pmu_cpu_dying,
..	..	@@ -3682,8 +4343,29 @@
3682	4343
3683	4344	.guest_get_msrs = intel_guest_get_msrs,
3684	4345	.sched_task = intel_pmu_sched_task,
	4346	+ .swap_task_ctx = intel_pmu_swap_task_ctx,
3685	4347
3686	4348	.check_period = intel_pmu_check_period,
	4349	+
	4350	+ .aux_output_match = intel_pmu_aux_output_match,
	4351	+
	4352	+ .lbr_reset = intel_pmu_lbr_reset_64,
	4353	+ .lbr_read = intel_pmu_lbr_read_64,
	4354	+ .lbr_save = intel_pmu_lbr_save,
	4355	+ .lbr_restore = intel_pmu_lbr_restore,
	4356	+
	4357	+ /*
	4358	+ * SMM has access to all 4 rings and while traditionally SMM code only
	4359	+ * ran in CPL0, 2021-era firmware is starting to make use of CPL3 in SMM.
	4360	+ *
	4361	+ * Since the EVENTSEL.{USR,OS} CPL filtering makes no distinction
	4362	+ * between SMM or not, this results in what should be pure userspace
	4363	+ * counters including SMM data.
	4364	+ *
	4365	+ * This is a clear privilege issue, therefore globally disable
	4366	+ * counting SMM by default.
	4367	+ */
	4368	+ .attr_freeze_on_smi = 1,
3687	4369	};
3688	4370
3689	4371	static __init void intel_clovertown_quirk(void)
..	..	@@ -3712,36 +4394,66 @@
3712	4394	x86_pmu.pebs_constraints = NULL;
3713	4395	}
3714	4396
3715		-static int intel_snb_pebs_broken(int cpu)
	4397	+static const struct x86_cpu_desc isolation_ucodes[] = {
	4398	+ INTEL_CPU_DESC(INTEL_FAM6_HASWELL, 3, 0x0000001f),
	4399	+ INTEL_CPU_DESC(INTEL_FAM6_HASWELL_L, 1, 0x0000001e),
	4400	+ INTEL_CPU_DESC(INTEL_FAM6_HASWELL_G, 1, 0x00000015),
	4401	+ INTEL_CPU_DESC(INTEL_FAM6_HASWELL_X, 2, 0x00000037),
	4402	+ INTEL_CPU_DESC(INTEL_FAM6_HASWELL_X, 4, 0x0000000a),
	4403	+ INTEL_CPU_DESC(INTEL_FAM6_BROADWELL, 4, 0x00000023),
	4404	+ INTEL_CPU_DESC(INTEL_FAM6_BROADWELL_G, 1, 0x00000014),
	4405	+ INTEL_CPU_DESC(INTEL_FAM6_BROADWELL_D, 2, 0x00000010),
	4406	+ INTEL_CPU_DESC(INTEL_FAM6_BROADWELL_D, 3, 0x07000009),
	4407	+ INTEL_CPU_DESC(INTEL_FAM6_BROADWELL_D, 4, 0x0f000009),
	4408	+ INTEL_CPU_DESC(INTEL_FAM6_BROADWELL_D, 5, 0x0e000002),
	4409	+ INTEL_CPU_DESC(INTEL_FAM6_BROADWELL_X, 1, 0x0b000014),
	4410	+ INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE_X, 3, 0x00000021),
	4411	+ INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE_X, 4, 0x00000000),
	4412	+ INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE_X, 5, 0x00000000),
	4413	+ INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE_X, 6, 0x00000000),
	4414	+ INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE_X, 7, 0x00000000),
	4415	+ INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE_X, 11, 0x00000000),
	4416	+ INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE_L, 3, 0x0000007c),
	4417	+ INTEL_CPU_DESC(INTEL_FAM6_SKYLAKE, 3, 0x0000007c),
	4418	+ INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE, 9, 0x0000004e),
	4419	+ INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE_L, 9, 0x0000004e),
	4420	+ INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE_L, 10, 0x0000004e),
	4421	+ INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE_L, 11, 0x0000004e),
	4422	+ INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE_L, 12, 0x0000004e),
	4423	+ INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE, 10, 0x0000004e),
	4424	+ INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE, 11, 0x0000004e),
	4425	+ INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE, 12, 0x0000004e),
	4426	+ INTEL_CPU_DESC(INTEL_FAM6_KABYLAKE, 13, 0x0000004e),
	4427	+ {}
	4428	+};
	4429	+
	4430	+static void intel_check_pebs_isolation(void)
3716	4431	{
3717		- u32 rev = UINT_MAX; /* default to broken for unknown models */
	4432	+ x86_pmu.pebs_no_isolation = !x86_cpu_has_min_microcode_rev(isolation_ucodes);
	4433	+}
3718	4434
3719		- switch (cpu_data(cpu).x86_model) {
3720		- case INTEL_FAM6_SANDYBRIDGE:
3721		- rev = 0x28;
3722		- break;
	4435	+static __init void intel_pebs_isolation_quirk(void)
	4436	+{
	4437	+ WARN_ON_ONCE(x86_pmu.check_microcode);
	4438	+ x86_pmu.check_microcode = intel_check_pebs_isolation;
	4439	+ intel_check_pebs_isolation();
	4440	+}
3723	4441
3724		- case INTEL_FAM6_SANDYBRIDGE_X:
3725		- switch (cpu_data(cpu).x86_stepping) {
3726		- case 6: rev = 0x618; break;
3727		- case 7: rev = 0x70c; break;
3728		- }
3729		- }
	4442	+static const struct x86_cpu_desc pebs_ucodes[] = {
	4443	+ INTEL_CPU_DESC(INTEL_FAM6_SANDYBRIDGE, 7, 0x00000028),
	4444	+ INTEL_CPU_DESC(INTEL_FAM6_SANDYBRIDGE_X, 6, 0x00000618),
	4445	+ INTEL_CPU_DESC(INTEL_FAM6_SANDYBRIDGE_X, 7, 0x0000070c),
	4446	+ {}
	4447	+};
3730	4448
3731		- return (cpu_data(cpu).microcode < rev);
	4449	+static bool intel_snb_pebs_broken(void)
	4450	+{
	4451	+ return !x86_cpu_has_min_microcode_rev(pebs_ucodes);
3732	4452	}
3733	4453
3734	4454	static void intel_snb_check_microcode(void)
3735	4455	{
3736		- int pebs_broken = 0;
3737		- int cpu;
3738		-
3739		- for_each_online_cpu(cpu) {
3740		- if ((pebs_broken = intel_snb_pebs_broken(cpu)))
3741		- break;
3742		- }
3743		-
3744		- if (pebs_broken == x86_pmu.pebs_broken)
	4456	+ if (intel_snb_pebs_broken() == x86_pmu.pebs_broken)
3745	4457	return;
3746	4458
3747	4459	/*
..	..	@@ -3770,6 +4482,13 @@
3770	4482	static bool check_msr(unsigned long msr, u64 mask)
3771	4483	{
3772	4484	u64 val_old, val_new, val_tmp;
	4485	+
	4486	+ /*
	4487	+ * Disable the check for real HW, so we don't
	4488	+ * mess with potentionaly enabled registers:
	4489	+ */
	4490	+ if (!boot_cpu_has(X86_FEATURE_HYPERVISOR))
	4491	+ return true;
3773	4492
3774	4493	/*
3775	4494	* Read the current value, change it and read it back to see if it
..	..	@@ -3858,6 +4577,39 @@
3858	4577	}
3859	4578	}
3860	4579
	4580	+static const struct x86_cpu_desc counter_freezing_ucodes[] = {
	4581	+ INTEL_CPU_DESC(INTEL_FAM6_ATOM_GOLDMONT, 2, 0x0000000e),
	4582	+ INTEL_CPU_DESC(INTEL_FAM6_ATOM_GOLDMONT, 9, 0x0000002e),
	4583	+ INTEL_CPU_DESC(INTEL_FAM6_ATOM_GOLDMONT, 10, 0x00000008),
	4584	+ INTEL_CPU_DESC(INTEL_FAM6_ATOM_GOLDMONT_D, 1, 0x00000028),
	4585	+ INTEL_CPU_DESC(INTEL_FAM6_ATOM_GOLDMONT_PLUS, 1, 0x00000028),
	4586	+ INTEL_CPU_DESC(INTEL_FAM6_ATOM_GOLDMONT_PLUS, 8, 0x00000006),
	4587	+ {}
	4588	+};
	4589	+
	4590	+static bool intel_counter_freezing_broken(void)
	4591	+{
	4592	+ return !x86_cpu_has_min_microcode_rev(counter_freezing_ucodes);
	4593	+}
	4594	+
	4595	+static __init void intel_counter_freezing_quirk(void)
	4596	+{
	4597	+ /* Check if it's already disabled */
	4598	+ if (disable_counter_freezing)
	4599	+ return;
	4600	+
	4601	+ /*
	4602	+ * If the system starts with the wrong ucode, leave the
	4603	+ * counter-freezing feature permanently disabled.
	4604	+ */
	4605	+ if (intel_counter_freezing_broken()) {
	4606	+ pr_info("PMU counter freezing disabled due to CPU errata,"
	4607	+ "please upgrade microcode\n");
	4608	+ x86_pmu.counter_freezing = false;
	4609	+ x86_pmu.handle_irq = intel_pmu_handle_irq;
	4610	+ }
	4611	+}
	4612	+
3861	4613	/*
3862	4614	* enable software workaround for errata:
3863	4615	* SNB: BJ122
..	..	@@ -3897,8 +4649,6 @@
3897	4649	EVENT_ATTR_STR(cycles-ct, cycles_ct, "event=0x3c,in_tx=1,in_tx_cp=1");
3898	4650
3899	4651	static struct attribute *hsw_events_attrs[] = {
3900		- EVENT_PTR(mem_ld_hsw),
3901		- EVENT_PTR(mem_st_hsw),
3902	4652	EVENT_PTR(td_slots_issued),
3903	4653	EVENT_PTR(td_slots_retired),
3904	4654	EVENT_PTR(td_fetch_bubbles),
..	..	@@ -3907,6 +4657,12 @@
3907	4657	EVENT_PTR(td_recovery_bubbles),
3908	4658	EVENT_PTR(td_recovery_bubbles_scale),
3909	4659	NULL
	4660	+};
	4661	+
	4662	+static struct attribute *hsw_mem_events_attrs[] = {
	4663	+ EVENT_PTR(mem_ld_hsw),
	4664	+ EVENT_PTR(mem_st_hsw),
	4665	+ NULL,
3910	4666	};
3911	4667
3912	4668	static struct attribute *hsw_tsx_events_attrs[] = {
..	..	@@ -3925,12 +4681,43 @@
3925	4681	NULL
3926	4682	};
3927	4683
3928		-static __init struct attribute **get_hsw_events_attrs(void)
3929		-{
3930		- return boot_cpu_has(X86_FEATURE_RTM) ?
3931		- merge_attr(hsw_events_attrs, hsw_tsx_events_attrs) :
3932		- hsw_events_attrs;
3933		-}
	4684	+EVENT_ATTR_STR(tx-capacity-read, tx_capacity_read, "event=0x54,umask=0x80");
	4685	+EVENT_ATTR_STR(tx-capacity-write, tx_capacity_write, "event=0x54,umask=0x2");
	4686	+EVENT_ATTR_STR(el-capacity-read, el_capacity_read, "event=0x54,umask=0x80");
	4687	+EVENT_ATTR_STR(el-capacity-write, el_capacity_write, "event=0x54,umask=0x2");
	4688	+
	4689	+static struct attribute *icl_events_attrs[] = {
	4690	+ EVENT_PTR(mem_ld_hsw),
	4691	+ EVENT_PTR(mem_st_hsw),
	4692	+ NULL,
	4693	+};
	4694	+
	4695	+static struct attribute *icl_td_events_attrs[] = {
	4696	+ EVENT_PTR(slots),
	4697	+ EVENT_PTR(td_retiring),
	4698	+ EVENT_PTR(td_bad_spec),
	4699	+ EVENT_PTR(td_fe_bound),
	4700	+ EVENT_PTR(td_be_bound),
	4701	+ NULL,
	4702	+};
	4703	+
	4704	+static struct attribute *icl_tsx_events_attrs[] = {
	4705	+ EVENT_PTR(tx_start),
	4706	+ EVENT_PTR(tx_abort),
	4707	+ EVENT_PTR(tx_commit),
	4708	+ EVENT_PTR(tx_capacity_read),
	4709	+ EVENT_PTR(tx_capacity_write),
	4710	+ EVENT_PTR(tx_conflict),
	4711	+ EVENT_PTR(el_start),
	4712	+ EVENT_PTR(el_abort),
	4713	+ EVENT_PTR(el_commit),
	4714	+ EVENT_PTR(el_capacity_read),
	4715	+ EVENT_PTR(el_capacity_write),
	4716	+ EVENT_PTR(el_conflict),
	4717	+ EVENT_PTR(cycles_t),
	4718	+ EVENT_PTR(cycles_ct),
	4719	+ NULL,
	4720	+};
3934	4721
3935	4722	static ssize_t freeze_on_smi_show(struct device *cdev,
3936	4723	struct device_attribute *attr,
..	..	@@ -3971,6 +4758,50 @@
3971	4758	return count;
3972	4759	}
3973	4760
	4761	+static void update_tfa_sched(void *ignored)
	4762	+{
	4763	+ struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
	4764	+
	4765	+ /*
	4766	+ * check if PMC3 is used
	4767	+ * and if so force schedule out for all event types all contexts
	4768	+ */
	4769	+ if (test_bit(3, cpuc->active_mask))
	4770	+ perf_pmu_resched(x86_get_pmu(smp_processor_id()));
	4771	+}
	4772	+
	4773	+static ssize_t show_sysctl_tfa(struct device *cdev,
	4774	+ struct device_attribute *attr,
	4775	+ char *buf)
	4776	+{
	4777	+ return snprintf(buf, 40, "%d\n", allow_tsx_force_abort);
	4778	+}
	4779	+
	4780	+static ssize_t set_sysctl_tfa(struct device *cdev,
	4781	+ struct device_attribute *attr,
	4782	+ const char *buf, size_t count)
	4783	+{
	4784	+ bool val;
	4785	+ ssize_t ret;
	4786	+
	4787	+ ret = kstrtobool(buf, &val);
	4788	+ if (ret)
	4789	+ return ret;
	4790	+
	4791	+ /* no change */
	4792	+ if (val == allow_tsx_force_abort)
	4793	+ return count;
	4794	+
	4795	+ allow_tsx_force_abort = val;
	4796	+
	4797	+ get_online_cpus();
	4798	+ on_each_cpu(update_tfa_sched, NULL, 1);
	4799	+ put_online_cpus();
	4800	+
	4801	+ return count;
	4802	+}
	4803	+
	4804	+
3974	4805	static DEVICE_ATTR_RW(freeze_on_smi);
3975	4806
3976	4807	static ssize_t branches_show(struct device *cdev,
..	..	@@ -4003,24 +4834,117 @@
4003	4834	NULL
4004	4835	};
4005	4836
4006		-static DEVICE_BOOL_ATTR(allow_tsx_force_abort, 0644, allow_tsx_force_abort);
	4837	+static DEVICE_ATTR(allow_tsx_force_abort, 0644,
	4838	+ show_sysctl_tfa,
	4839	+ set_sysctl_tfa);
4007	4840
4008	4841	static struct attribute *intel_pmu_attrs[] = {
4009	4842	&dev_attr_freeze_on_smi.attr,
4010		- NULL, /* &dev_attr_allow_tsx_force_abort.attr.attr */
	4843	+ &dev_attr_allow_tsx_force_abort.attr,
4011	4844	NULL,
4012	4845	};
4013	4846
	4847	+static umode_t
	4848	+tsx_is_visible(struct kobject kobj, struct attribute attr, int i)
	4849	+{
	4850	+ return boot_cpu_has(X86_FEATURE_RTM) ? attr->mode : 0;
	4851	+}
	4852	+
	4853	+static umode_t
	4854	+pebs_is_visible(struct kobject kobj, struct attribute attr, int i)
	4855	+{
	4856	+ return x86_pmu.pebs ? attr->mode : 0;
	4857	+}
	4858	+
	4859	+static umode_t
	4860	+lbr_is_visible(struct kobject kobj, struct attribute attr, int i)
	4861	+{
	4862	+ return x86_pmu.lbr_nr ? attr->mode : 0;
	4863	+}
	4864	+
	4865	+static umode_t
	4866	+exra_is_visible(struct kobject kobj, struct attribute attr, int i)
	4867	+{
	4868	+ return x86_pmu.version >= 2 ? attr->mode : 0;
	4869	+}
	4870	+
	4871	+static umode_t
	4872	+default_is_visible(struct kobject kobj, struct attribute attr, int i)
	4873	+{
	4874	+ if (attr == &dev_attr_allow_tsx_force_abort.attr)
	4875	+ return x86_pmu.flags & PMU_FL_TFA ? attr->mode : 0;
	4876	+
	4877	+ return attr->mode;
	4878	+}
	4879	+
	4880	+static struct attribute_group group_events_td = {
	4881	+ .name = "events",
	4882	+};
	4883	+
	4884	+static struct attribute_group group_events_mem = {
	4885	+ .name = "events",
	4886	+ .is_visible = pebs_is_visible,
	4887	+};
	4888	+
	4889	+static struct attribute_group group_events_tsx = {
	4890	+ .name = "events",
	4891	+ .is_visible = tsx_is_visible,
	4892	+};
	4893	+
	4894	+static struct attribute_group group_caps_gen = {
	4895	+ .name = "caps",
	4896	+ .attrs = intel_pmu_caps_attrs,
	4897	+};
	4898	+
	4899	+static struct attribute_group group_caps_lbr = {
	4900	+ .name = "caps",
	4901	+ .attrs = lbr_attrs,
	4902	+ .is_visible = lbr_is_visible,
	4903	+};
	4904	+
	4905	+static struct attribute_group group_format_extra = {
	4906	+ .name = "format",
	4907	+ .is_visible = exra_is_visible,
	4908	+};
	4909	+
	4910	+static struct attribute_group group_format_extra_skl = {
	4911	+ .name = "format",
	4912	+ .is_visible = exra_is_visible,
	4913	+};
	4914	+
	4915	+static struct attribute_group group_default = {
	4916	+ .attrs = intel_pmu_attrs,
	4917	+ .is_visible = default_is_visible,
	4918	+};
	4919	+
	4920	+static const struct attribute_group *attr_update[] = {
	4921	+ &group_events_td,
	4922	+ &group_events_mem,
	4923	+ &group_events_tsx,
	4924	+ &group_caps_gen,
	4925	+ &group_caps_lbr,
	4926	+ &group_format_extra,
	4927	+ &group_format_extra_skl,
	4928	+ &group_default,
	4929	+ NULL,
	4930	+};
	4931	+
	4932	+static struct attribute *empty_attrs;
	4933	+
4014	4934	__init int intel_pmu_init(void)
4015	4935	{
4016		- struct attribute **extra_attr = NULL;
4017		- struct attribute **to_free = NULL;
	4936	+ struct attribute **extra_skl_attr = &empty_attrs;
	4937	+ struct attribute **extra_attr = &empty_attrs;
	4938	+ struct attribute **td_attr = &empty_attrs;
	4939	+ struct attribute **mem_attr = &empty_attrs;
	4940	+ struct attribute **tsx_attr = &empty_attrs;
4018	4941	union cpuid10_edx edx;
4019	4942	union cpuid10_eax eax;
4020	4943	union cpuid10_ebx ebx;
4021	4944	struct event_constraint *c;
4022	4945	unsigned int unused;
4023	4946	struct extra_reg *er;
	4947	+ bool pmem = false;
4024	4948	int version, i;
4025	4949	char *name;
4026	4950
..	..	@@ -4071,6 +4995,9 @@
4071	4995	max((int)edx.split.num_counters_fixed, assume);
4072	4996	}
4073	4997
	4998	+ if (version >= 4)
	4999	+ x86_pmu.counter_freezing = !disable_counter_freezing;
	5000	+
4074	5001	if (boot_cpu_has(X86_FEATURE_PDCM)) {
4075	5002	u64 capabilities;
4076	5003
..	..	@@ -4078,9 +5005,23 @@
4078	5005	x86_pmu.intel_cap.capabilities = capabilities;
4079	5006	}
4080	5007
	5008	+ if (x86_pmu.intel_cap.lbr_format == LBR_FORMAT_32) {
	5009	+ x86_pmu.lbr_reset = intel_pmu_lbr_reset_32;
	5010	+ x86_pmu.lbr_read = intel_pmu_lbr_read_32;
	5011	+ }
	5012	+
	5013	+ if (boot_cpu_has(X86_FEATURE_ARCH_LBR))
	5014	+ intel_pmu_arch_lbr_init();
	5015	+
4081	5016	intel_ds_init();
4082	5017
4083	5018	x86_add_quirk(intel_arch_events_quirk); /* Install first, so it runs last */
	5019	+
	5020	+ if (version >= 5) {
	5021	+ x86_pmu.intel_cap.anythread_deprecated = edx.split.anythread_deprecated;
	5022	+ if (x86_pmu.intel_cap.anythread_deprecated)
	5023	+ pr_cont(" AnyThread deprecated, ");
	5024	+ }
4084	5025
4085	5026	/*
4086	5027	* Install the hw-cache-events table:
..	..	@@ -4093,6 +5034,8 @@
4093	5034
4094	5035	case INTEL_FAM6_CORE2_MEROM:
4095	5036	x86_add_quirk(intel_clovertown_quirk);
	5037	+ fallthrough;
	5038	+
4096	5039	case INTEL_FAM6_CORE2_MEROM_L:
4097	5040	case INTEL_FAM6_CORE2_PENRYN:
4098	5041	case INTEL_FAM6_CORE2_DUNNINGTON:
..	..	@@ -4123,7 +5066,7 @@
4123	5066	x86_pmu.extra_regs = intel_nehalem_extra_regs;
4124	5067	x86_pmu.limit_period = nhm_limit_period;
4125	5068
4126		- x86_pmu.cpu_events = nhm_events_attrs;
	5069	+ mem_attr = nhm_mem_events_attrs;
4127	5070
4128	5071	/* UOPS_ISSUED.STALLED_CYCLES */
4129	5072	intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
..	..	@@ -4159,7 +5102,7 @@
4159	5102	break;
4160	5103
4161	5104	case INTEL_FAM6_ATOM_SILVERMONT:
4162		- case INTEL_FAM6_ATOM_SILVERMONT_X:
	5105	+ case INTEL_FAM6_ATOM_SILVERMONT_D:
4163	5106	case INTEL_FAM6_ATOM_SILVERMONT_MID:
4164	5107	case INTEL_FAM6_ATOM_AIRMONT:
4165	5108	case INTEL_FAM6_ATOM_AIRMONT_MID:
..	..	@@ -4174,14 +5117,15 @@
4174	5117	x86_pmu.pebs_constraints = intel_slm_pebs_event_constraints;
4175	5118	x86_pmu.extra_regs = intel_slm_extra_regs;
4176	5119	x86_pmu.flags \|= PMU_FL_HAS_RSP_1;
4177		- x86_pmu.cpu_events = slm_events_attrs;
	5120	+ td_attr = slm_events_attrs;
4178	5121	extra_attr = slm_format_attr;
4179	5122	pr_cont("Silvermont events, ");
4180	5123	name = "silvermont";
4181	5124	break;
4182	5125
4183	5126	case INTEL_FAM6_ATOM_GOLDMONT:
4184		- case INTEL_FAM6_ATOM_GOLDMONT_X:
	5127	+ case INTEL_FAM6_ATOM_GOLDMONT_D:
	5128	+ x86_add_quirk(intel_counter_freezing_quirk);
4185	5129	memcpy(hw_cache_event_ids, glm_hw_cache_event_ids,
4186	5130	sizeof(hw_cache_event_ids));
4187	5131	memcpy(hw_cache_extra_regs, glm_hw_cache_extra_regs,
..	..	@@ -4201,13 +5145,14 @@
4201	5145	x86_pmu.pebs_prec_dist = true;
4202	5146	x86_pmu.lbr_pt_coexist = true;
4203	5147	x86_pmu.flags \|= PMU_FL_HAS_RSP_1;
4204		- x86_pmu.cpu_events = glm_events_attrs;
	5148	+ td_attr = glm_events_attrs;
4205	5149	extra_attr = slm_format_attr;
4206	5150	pr_cont("Goldmont events, ");
4207	5151	name = "goldmont";
4208	5152	break;
4209	5153
4210	5154	case INTEL_FAM6_ATOM_GOLDMONT_PLUS:
	5155	+ x86_add_quirk(intel_counter_freezing_quirk);
4211	5156	memcpy(hw_cache_event_ids, glp_hw_cache_event_ids,
4212	5157	sizeof(hw_cache_event_ids));
4213	5158	memcpy(hw_cache_extra_regs, glp_hw_cache_extra_regs,
..	..	@@ -4227,12 +5172,40 @@
4227	5172	x86_pmu.flags \|= PMU_FL_HAS_RSP_1;
4228	5173	x86_pmu.flags \|= PMU_FL_PEBS_ALL;
4229	5174	x86_pmu.get_event_constraints = glp_get_event_constraints;
4230		- x86_pmu.cpu_events = glm_events_attrs;
	5175	+ td_attr = glm_events_attrs;
4231	5176	/* Goldmont Plus has 4-wide pipeline */
4232	5177	event_attr_td_total_slots_scale_glm.event_str = "4";
4233	5178	extra_attr = slm_format_attr;
4234	5179	pr_cont("Goldmont plus events, ");
4235	5180	name = "goldmont_plus";
	5181	+ break;
	5182	+
	5183	+ case INTEL_FAM6_ATOM_TREMONT_D:
	5184	+ case INTEL_FAM6_ATOM_TREMONT:
	5185	+ case INTEL_FAM6_ATOM_TREMONT_L:
	5186	+ x86_pmu.late_ack = true;
	5187	+ memcpy(hw_cache_event_ids, glp_hw_cache_event_ids,
	5188	+ sizeof(hw_cache_event_ids));
	5189	+ memcpy(hw_cache_extra_regs, tnt_hw_cache_extra_regs,
	5190	+ sizeof(hw_cache_extra_regs));
	5191	+ hw_cache_event_ids[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = -1;
	5192	+
	5193	+ intel_pmu_lbr_init_skl();
	5194	+
	5195	+ x86_pmu.event_constraints = intel_slm_event_constraints;
	5196	+ x86_pmu.extra_regs = intel_tnt_extra_regs;
	5197	+ /*
	5198	+ * It's recommended to use CPU_CLK_UNHALTED.CORE_P + NPEBS
	5199	+ * for precise cycles.
	5200	+ */
	5201	+ x86_pmu.pebs_aliases = NULL;
	5202	+ x86_pmu.pebs_prec_dist = true;
	5203	+ x86_pmu.lbr_pt_coexist = true;
	5204	+ x86_pmu.flags \|= PMU_FL_HAS_RSP_1;
	5205	+ x86_pmu.get_event_constraints = tnt_get_event_constraints;
	5206	+ extra_attr = slm_format_attr;
	5207	+ pr_cont("Tremont events, ");
	5208	+ name = "Tremont";
4236	5209	break;
4237	5210
4238	5211	case INTEL_FAM6_WESTMERE:
..	..	@@ -4251,7 +5224,7 @@
4251	5224	x86_pmu.extra_regs = intel_westmere_extra_regs;
4252	5225	x86_pmu.flags \|= PMU_FL_HAS_RSP_1;
4253	5226
4254		- x86_pmu.cpu_events = nhm_events_attrs;
	5227	+ mem_attr = nhm_mem_events_attrs;
4255	5228
4256	5229	/* UOPS_ISSUED.STALLED_CYCLES */
4257	5230	intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
..	..	@@ -4290,7 +5263,8 @@
4290	5263	x86_pmu.flags \|= PMU_FL_HAS_RSP_1;
4291	5264	x86_pmu.flags \|= PMU_FL_NO_HT_SHARING;
4292	5265
4293		- x86_pmu.cpu_events = snb_events_attrs;
	5266	+ td_attr = snb_events_attrs;
	5267	+ mem_attr = snb_mem_events_attrs;
4294	5268
4295	5269	/* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */
4296	5270	intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
..	..	@@ -4330,7 +5304,8 @@
4330	5304	x86_pmu.flags \|= PMU_FL_HAS_RSP_1;
4331	5305	x86_pmu.flags \|= PMU_FL_NO_HT_SHARING;
4332	5306
4333		- x86_pmu.cpu_events = snb_events_attrs;
	5307	+ td_attr = snb_events_attrs;
	5308	+ mem_attr = snb_mem_events_attrs;
4334	5309
4335	5310	/* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */
4336	5311	intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
..	..	@@ -4343,11 +5318,12 @@
4343	5318	break;
4344	5319
4345	5320
4346		- case INTEL_FAM6_HASWELL_CORE:
	5321	+ case INTEL_FAM6_HASWELL:
4347	5322	case INTEL_FAM6_HASWELL_X:
4348		- case INTEL_FAM6_HASWELL_ULT:
4349		- case INTEL_FAM6_HASWELL_GT3E:
	5323	+ case INTEL_FAM6_HASWELL_L:
	5324	+ case INTEL_FAM6_HASWELL_G:
4350	5325	x86_add_quirk(intel_ht_bug);
	5326	+ x86_add_quirk(intel_pebs_isolation_quirk);
4351	5327	x86_pmu.late_ack = true;
4352	5328	memcpy(hw_cache_event_ids, hsw_hw_cache_event_ids, sizeof(hw_cache_event_ids));
4353	5329	memcpy(hw_cache_extra_regs, hsw_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
..	..	@@ -4365,18 +5341,21 @@
4365	5341
4366	5342	x86_pmu.hw_config = hsw_hw_config;
4367	5343	x86_pmu.get_event_constraints = hsw_get_event_constraints;
4368		- x86_pmu.cpu_events = get_hsw_events_attrs();
4369	5344	x86_pmu.lbr_double_abort = true;
4370	5345	extra_attr = boot_cpu_has(X86_FEATURE_RTM) ?
4371	5346	hsw_format_attr : nhm_format_attr;
	5347	+ td_attr = hsw_events_attrs;
	5348	+ mem_attr = hsw_mem_events_attrs;
	5349	+ tsx_attr = hsw_tsx_events_attrs;
4372	5350	pr_cont("Haswell events, ");
4373	5351	name = "haswell";
4374	5352	break;
4375	5353
4376		- case INTEL_FAM6_BROADWELL_CORE:
4377		- case INTEL_FAM6_BROADWELL_XEON_D:
4378		- case INTEL_FAM6_BROADWELL_GT3E:
	5354	+ case INTEL_FAM6_BROADWELL:
	5355	+ case INTEL_FAM6_BROADWELL_D:
	5356	+ case INTEL_FAM6_BROADWELL_G:
4379	5357	case INTEL_FAM6_BROADWELL_X:
	5358	+ x86_add_quirk(intel_pebs_isolation_quirk);
4380	5359	x86_pmu.late_ack = true;
4381	5360	memcpy(hw_cache_event_ids, hsw_hw_cache_event_ids, sizeof(hw_cache_event_ids));
4382	5361	memcpy(hw_cache_extra_regs, hsw_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
..	..	@@ -4404,10 +5383,12 @@
4404	5383
4405	5384	x86_pmu.hw_config = hsw_hw_config;
4406	5385	x86_pmu.get_event_constraints = hsw_get_event_constraints;
4407		- x86_pmu.cpu_events = get_hsw_events_attrs();
4408	5386	x86_pmu.limit_period = bdw_limit_period;
4409	5387	extra_attr = boot_cpu_has(X86_FEATURE_RTM) ?
4410	5388	hsw_format_attr : nhm_format_attr;
	5389	+ td_attr = hsw_events_attrs;
	5390	+ mem_attr = hsw_mem_events_attrs;
	5391	+ tsx_attr = hsw_tsx_events_attrs;
4411	5392	pr_cont("Broadwell events, ");
4412	5393	name = "broadwell";
4413	5394	break;
..	..	@@ -4432,11 +5413,16 @@
4432	5413	name = "knights-landing";
4433	5414	break;
4434	5415
4435		- case INTEL_FAM6_SKYLAKE_MOBILE:
4436		- case INTEL_FAM6_SKYLAKE_DESKTOP:
4437	5416	case INTEL_FAM6_SKYLAKE_X:
4438		- case INTEL_FAM6_KABYLAKE_MOBILE:
4439		- case INTEL_FAM6_KABYLAKE_DESKTOP:
	5417	+ pmem = true;
	5418	+ fallthrough;
	5419	+ case INTEL_FAM6_SKYLAKE_L:
	5420	+ case INTEL_FAM6_SKYLAKE:
	5421	+ case INTEL_FAM6_KABYLAKE_L:
	5422	+ case INTEL_FAM6_KABYLAKE:
	5423	+ case INTEL_FAM6_COMETLAKE_L:
	5424	+ case INTEL_FAM6_COMETLAKE:
	5425	+ x86_add_quirk(intel_pebs_isolation_quirk);
4440	5426	x86_pmu.late_ack = true;
4441	5427	memcpy(hw_cache_event_ids, skl_hw_cache_event_ids, sizeof(hw_cache_event_ids));
4442	5428	memcpy(hw_cache_extra_regs, skl_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
..	..	@@ -4461,22 +5447,60 @@
4461	5447	x86_pmu.get_event_constraints = hsw_get_event_constraints;
4462	5448	extra_attr = boot_cpu_has(X86_FEATURE_RTM) ?
4463	5449	hsw_format_attr : nhm_format_attr;
4464		- extra_attr = merge_attr(extra_attr, skl_format_attr);
4465		- to_free = extra_attr;
4466		- x86_pmu.cpu_events = get_hsw_events_attrs();
4467		- intel_pmu_pebs_data_source_skl(
4468		- boot_cpu_data.x86_model == INTEL_FAM6_SKYLAKE_X);
	5450	+ extra_skl_attr = skl_format_attr;
	5451	+ td_attr = hsw_events_attrs;
	5452	+ mem_attr = hsw_mem_events_attrs;
	5453	+ tsx_attr = hsw_tsx_events_attrs;
	5454	+ intel_pmu_pebs_data_source_skl(pmem);
4469	5455
4470	5456	if (boot_cpu_has(X86_FEATURE_TSX_FORCE_ABORT)) {
4471	5457	x86_pmu.flags \|= PMU_FL_TFA;
4472	5458	x86_pmu.get_event_constraints = tfa_get_event_constraints;
4473	5459	x86_pmu.enable_all = intel_tfa_pmu_enable_all;
4474	5460	x86_pmu.commit_scheduling = intel_tfa_commit_scheduling;
4475		- intel_pmu_attrs[1] = &dev_attr_allow_tsx_force_abort.attr.attr;
4476	5461	}
4477	5462
4478	5463	pr_cont("Skylake events, ");
4479	5464	name = "skylake";
	5465	+ break;
	5466	+
	5467	+ case INTEL_FAM6_ICELAKE_X:
	5468	+ case INTEL_FAM6_ICELAKE_D:
	5469	+ pmem = true;
	5470	+ fallthrough;
	5471	+ case INTEL_FAM6_ICELAKE_L:
	5472	+ case INTEL_FAM6_ICELAKE:
	5473	+ case INTEL_FAM6_TIGERLAKE_L:
	5474	+ case INTEL_FAM6_TIGERLAKE:
	5475	+ x86_pmu.late_ack = true;
	5476	+ memcpy(hw_cache_event_ids, skl_hw_cache_event_ids, sizeof(hw_cache_event_ids));
	5477	+ memcpy(hw_cache_extra_regs, skl_hw_cache_extra_regs, sizeof(hw_cache_extra_regs));
	5478	+ hw_cache_event_ids[C(ITLB)][C(OP_READ)][C(RESULT_ACCESS)] = -1;
	5479	+ intel_pmu_lbr_init_skl();
	5480	+
	5481	+ x86_pmu.event_constraints = intel_icl_event_constraints;
	5482	+ x86_pmu.pebs_constraints = intel_icl_pebs_event_constraints;
	5483	+ x86_pmu.extra_regs = intel_icl_extra_regs;
	5484	+ x86_pmu.pebs_aliases = NULL;
	5485	+ x86_pmu.pebs_prec_dist = true;
	5486	+ x86_pmu.flags \|= PMU_FL_HAS_RSP_1;
	5487	+ x86_pmu.flags \|= PMU_FL_NO_HT_SHARING;
	5488	+
	5489	+ x86_pmu.hw_config = hsw_hw_config;
	5490	+ x86_pmu.get_event_constraints = icl_get_event_constraints;
	5491	+ extra_attr = boot_cpu_has(X86_FEATURE_RTM) ?
	5492	+ hsw_format_attr : nhm_format_attr;
	5493	+ extra_skl_attr = skl_format_attr;
	5494	+ mem_attr = icl_events_attrs;
	5495	+ td_attr = icl_td_events_attrs;
	5496	+ tsx_attr = icl_tsx_events_attrs;
	5497	+ x86_pmu.rtm_abort_event = X86_CONFIG(.event=0xc9, .umask=0x04);
	5498	+ x86_pmu.lbr_pt_coexist = true;
	5499	+ intel_pmu_pebs_data_source_skl(pmem);
	5500	+ x86_pmu.update_topdown_event = icl_update_topdown_event;
	5501	+ x86_pmu.set_topdown_event_period = icl_set_topdown_event_period;
	5502	+ pr_cont("Icelake events, ");
	5503	+ name = "icelake";
4480	5504	break;
4481	5505
4482	5506	default:
..	..	@@ -4497,13 +5521,16 @@
4497	5521	}
4498	5522	}
4499	5523
4500		- snprintf(pmu_name_str, sizeof pmu_name_str, "%s", name);
	5524	+ snprintf(pmu_name_str, sizeof(pmu_name_str), "%s", name);
4501	5525
4502		- if (version >= 2 && extra_attr) {
4503		- x86_pmu.format_attrs = merge_attr(intel_arch3_formats_attr,
4504		- extra_attr);
4505		- WARN_ON(!x86_pmu.format_attrs);
4506		- }
	5526	+
	5527	+ group_events_td.attrs = td_attr;
	5528	+ group_events_mem.attrs = mem_attr;
	5529	+ group_events_tsx.attrs = tsx_attr;
	5530	+ group_format_extra.attrs = extra_attr;
	5531	+ group_format_extra_skl.attrs = extra_skl_attr;
	5532	+
	5533	+ x86_pmu.attr_update = attr_update;
4507	5534
4508	5535	if (x86_pmu.num_counters > INTEL_PMC_MAX_GENERIC) {
4509	5536	WARN(1, KERN_ERR "hw perf events %d > max(%d), clipping!",
..	..	@@ -4521,12 +5548,25 @@
4521	5548	x86_pmu.intel_ctrl \|=
4522	5549	((1LL << x86_pmu.num_counters_fixed)-1) << INTEL_PMC_IDX_FIXED;
4523	5550
	5551	+ /* AnyThread may be deprecated on arch perfmon v5 or later */
	5552	+ if (x86_pmu.intel_cap.anythread_deprecated)
	5553	+ x86_pmu.format_attrs = intel_arch_formats_attr;
	5554	+
4524	5555	if (x86_pmu.event_constraints) {
4525	5556	/*
4526	5557	* event on fixed counter2 (REF_CYCLES) only works on this
4527	5558	* counter, so do not extend mask to generic counters
4528	5559	*/
4529	5560	for_each_event_constraint(c, x86_pmu.event_constraints) {
	5561	+ /*
	5562	+ * Don't extend the topdown slots and metrics
	5563	+ * events to the generic counters.
	5564	+ */
	5565	+ if (c->idxmsk64 & INTEL_PMC_MSK_TOPDOWN) {
	5566	+ c->weight = hweight64(c->idxmsk64);
	5567	+ continue;
	5568	+ }
	5569	+
4530	5570	if (c->cmask == FIXED_EVENT_FLAGS
4531	5571	&& c->idxmsk64 != INTEL_PMC_MSK_FIXED_REF_CYCLES) {
4532	5572	c->idxmsk64 \|= (1ULL << x86_pmu.num_counters) - 1;
..	..	@@ -4543,7 +5583,7 @@
4543	5583	* Check all LBT MSR here.
4544	5584	* Disable LBR access if any LBR MSRs can not be accessed.
4545	5585	*/
4546		- if (x86_pmu.lbr_nr && !check_msr(x86_pmu.lbr_tos, 0x3UL))
	5586	+ if (x86_pmu.lbr_tos && !check_msr(x86_pmu.lbr_tos, 0x3UL))
4547	5587	x86_pmu.lbr_nr = 0;
4548	5588	for (i = 0; i < x86_pmu.lbr_nr; i++) {
4549	5589	if (!(check_msr(x86_pmu.lbr_from + i, 0xffffUL) &&
..	..	@@ -4551,12 +5591,8 @@
4551	5591	x86_pmu.lbr_nr = 0;
4552	5592	}
4553	5593
4554		- x86_pmu.caps_attrs = intel_pmu_caps_attrs;
4555		-
4556		- if (x86_pmu.lbr_nr) {
4557		- x86_pmu.caps_attrs = merge_attr(x86_pmu.caps_attrs, lbr_attrs);
	5594	+ if (x86_pmu.lbr_nr)
4558	5595	pr_cont("%d-deep LBR, ", x86_pmu.lbr_nr);
4559		- }
4560	5596
4561	5597	/*
4562	5598	* Access extra MSR may cause #GP under certain circumstances.
..	..	@@ -4579,7 +5615,16 @@
4579	5615	pr_cont("full-width counters, ");
4580	5616	}
4581	5617
4582		- kfree(to_free);
	5618	+ /*
	5619	+ * For arch perfmon 4 use counter freezing to avoid
	5620	+ * several MSR accesses in the PMI.
	5621	+ */
	5622	+ if (x86_pmu.counter_freezing)
	5623	+ x86_pmu.handle_irq = intel_pmu_handle_irq_v4;
	5624	+
	5625	+ if (x86_pmu.intel_cap.perf_metrics)
	5626	+ x86_pmu.intel_ctrl \|= 1ULL << GLOBAL_CTRL_EN_PERF_METRICS;
	5627	+
4583	5628	return 0;
4584	5629	}
4585	5630