forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-11 072de836f53be56a70cecf70b43ae43b7ce17376 
 kernel/arch/x86/events/core.c
....@@ -28,6 +28,7 @@
2828 #include <linux/bitops.h>
2929 #include <linux/device.h>
3030 #include <linux/nospec.h>
31
+#include <linux/static_call.h>
3132 
3233 #include <asm/apic.h>
3334 #include <asm/stacktrace.h>
....@@ -44,12 +45,43 @@
4445 #include "perf_event.h"
4546 
4647 struct x86_pmu x86_pmu __read_mostly;
48
+static struct pmu pmu;
4749 
4850 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
4951 	.enabled = 1,
52
+	.pmu = &pmu,
5053 };
5154 
55
+DEFINE_STATIC_KEY_FALSE(rdpmc_never_available_key);
5256 DEFINE_STATIC_KEY_FALSE(rdpmc_always_available_key);
57
+
58
+/*
59
+ * This here uses DEFINE_STATIC_CALL_NULL() to get a static_call defined
60
+ * from just a typename, as opposed to an actual function.
61
+ */
62
+DEFINE_STATIC_CALL_NULL(x86_pmu_handle_irq,  *x86_pmu.handle_irq);
63
+DEFINE_STATIC_CALL_NULL(x86_pmu_disable_all, *x86_pmu.disable_all);
64
+DEFINE_STATIC_CALL_NULL(x86_pmu_enable_all,  *x86_pmu.enable_all);
65
+DEFINE_STATIC_CALL_NULL(x86_pmu_enable,	     *x86_pmu.enable);
66
+DEFINE_STATIC_CALL_NULL(x86_pmu_disable,     *x86_pmu.disable);
67
+
68
+DEFINE_STATIC_CALL_NULL(x86_pmu_add,  *x86_pmu.add);
69
+DEFINE_STATIC_CALL_NULL(x86_pmu_del,  *x86_pmu.del);
70
+DEFINE_STATIC_CALL_NULL(x86_pmu_read, *x86_pmu.read);
71
+
72
+DEFINE_STATIC_CALL_NULL(x86_pmu_schedule_events,       *x86_pmu.schedule_events);
73
+DEFINE_STATIC_CALL_NULL(x86_pmu_get_event_constraints, *x86_pmu.get_event_constraints);
74
+DEFINE_STATIC_CALL_NULL(x86_pmu_put_event_constraints, *x86_pmu.put_event_constraints);
75
+
76
+DEFINE_STATIC_CALL_NULL(x86_pmu_start_scheduling,  *x86_pmu.start_scheduling);
77
+DEFINE_STATIC_CALL_NULL(x86_pmu_commit_scheduling, *x86_pmu.commit_scheduling);
78
+DEFINE_STATIC_CALL_NULL(x86_pmu_stop_scheduling,   *x86_pmu.stop_scheduling);
79
+
80
+DEFINE_STATIC_CALL_NULL(x86_pmu_sched_task,    *x86_pmu.sched_task);
81
+DEFINE_STATIC_CALL_NULL(x86_pmu_swap_task_ctx, *x86_pmu.swap_task_ctx);
82
+
83
+DEFINE_STATIC_CALL_NULL(x86_pmu_drain_pebs,   *x86_pmu.drain_pebs);
84
+DEFINE_STATIC_CALL_NULL(x86_pmu_pebs_aliases, *x86_pmu.pebs_aliases);
5385 
5486 u64 __read_mostly hw_cache_event_ids
5587 				[PERF_COUNT_HW_CACHE_MAX]
....@@ -70,11 +102,13 @@
70102 	struct hw_perf_event *hwc = &event->hw;
71103 	int shift = 64 - x86_pmu.cntval_bits;
72104 	u64 prev_raw_count, new_raw_count;
73
-	int idx = hwc->idx;
74105 	u64 delta;
75106 
76
-	if (idx == INTEL_PMC_IDX_FIXED_BTS)
107
+	if (unlikely(!hwc->event_base))
77108 		return 0;
109
+
110
+	if (unlikely(is_topdown_count(event)) && x86_pmu.update_topdown_event)
111
+		return x86_pmu.update_topdown_event(event);
78112 
79113 	/*
80114 	 * Careful: an NMI might modify the previous event value.
....@@ -340,10 +374,12 @@
340374 	if (!atomic_inc_not_zero(&pmc_refcount)) {
341375 		mutex_lock(&pmc_reserve_mutex);
342376 		if (atomic_read(&pmc_refcount) == 0) {
343
-			if (!reserve_pmc_hardware())
377
+			if (!reserve_pmc_hardware()) {
344378 				err = -EBUSY;
345
-			else
379
+			} else {
346380 				reserve_ds_buffers();
381
+				reserve_lbr_buffers();
382
+			}
347383 		}
348384 		if (!err)
349385 			atomic_inc(&pmc_refcount);
....@@ -358,6 +394,7 @@
358394 	if (atomic_dec_and_mutex_lock(&pmc_refcount, &pmc_reserve_mutex)) {
359395 		release_pmc_hardware();
360396 		release_ds_buffers();
397
+		release_lbr_buffers();
361398 		mutex_unlock(&pmc_reserve_mutex);
362399 	}
363400 }
....@@ -565,6 +602,21 @@
565602 			return -EINVAL;
566603 	}
567604 
605
+	/* sample_regs_user never support XMM registers */
606
+	if (unlikely(event->attr.sample_regs_user & PERF_REG_EXTENDED_MASK))
607
+		return -EINVAL;
608
+	/*
609
+	 * Besides the general purpose registers, XMM registers may
610
+	 * be collected in PEBS on some platforms, e.g. Icelake
611
+	 */
612
+	if (unlikely(event->attr.sample_regs_intr & PERF_REG_EXTENDED_MASK)) {
613
+		if (!(event->pmu->capabilities & PERF_PMU_CAP_EXTENDED_REGS))
614
+			return -EINVAL;
615
+
616
+		if (!event->attr.precise_ip)
617
+			return -EINVAL;
618
+	}
619
+
568620 	return x86_setup_perfctr(event);
569621 }
570622 
....@@ -602,6 +654,7 @@
602654 	int idx;
603655 
604656 	for (idx = 0; idx < x86_pmu.num_counters; idx++) {
657
+		struct hw_perf_event *hwc = &cpuc->events[idx]->hw;
605658 		u64 val;
606659 
607660 		if (!test_bit(idx, cpuc->active_mask))
....@@ -611,6 +664,8 @@
611664 			continue;
612665 		val &= ~ARCH_PERFMON_EVENTSEL_ENABLE;
613666 		wrmsrl(x86_pmu_config_addr(idx), val);
667
+		if (is_counter_pair(hwc))
668
+			wrmsrl(x86_pmu_config_addr(idx + 1), 0);
614669 	}
615670 }
616671 
....@@ -641,7 +696,7 @@
641696 	cpuc->enabled = 0;
642697 	barrier();
643698 
644
-	x86_pmu.disable_all();
699
+	static_call(x86_pmu_disable_all)();
645700 }
646701 
647702 void x86_pmu_enable_all(int added)
....@@ -659,13 +714,24 @@
659714 	}
660715 }
661716 
662
-static struct pmu pmu;
663
-
664717 static inline int is_x86_event(struct perf_event *event)
665718 {
666719 	return event->pmu == &pmu;
667720 }
668721 
722
+struct pmu *x86_get_pmu(unsigned int cpu)
723
+{
724
+	struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
725
+
726
+	/*
727
+	 * All CPUs of the hybrid type have been offline.
728
+	 * The x86_get_pmu() should not be invoked.
729
+	 */
730
+	if (WARN_ON_ONCE(!cpuc->pmu))
731
+		return &pmu;
732
+
733
+	return cpuc->pmu;
734
+}
669735 /*
670736  * Event scheduler state:
671737  *
....@@ -679,7 +745,7 @@
679745 	int	counter;	/* counter index */
680746 	int	unassigned;	/* number of events to be assigned left */
681747 	int	nr_gp;		/* number of GP counters used */
682
-	unsigned long used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
748
+	u64	used;
683749 };
684750 
685751 /* Total max is X86_PMC_IDX_MAX, but we are O(n!) limited */
....@@ -736,8 +802,12 @@
736802 	sched->saved_states--;
737803 	sched->state = sched->saved[sched->saved_states];
738804 
739
-	/* continue with next counter: */
740
-	clear_bit(sched->state.counter++, sched->state.used);
805
+	/* this assignment didn't work out */
806
+	/* XXX broken vs EVENT_PAIR */
807
+	sched->state.used &= ~BIT_ULL(sched->state.counter);
808
+
809
+	/* try the next one */
810
+	sched->state.counter++;
741811 
742812 	return true;
743813 }
....@@ -762,20 +832,32 @@
762832 	if (c->idxmsk64 & (~0ULL << INTEL_PMC_IDX_FIXED)) {
763833 		idx = INTEL_PMC_IDX_FIXED;
764834 		for_each_set_bit_from(idx, c->idxmsk, X86_PMC_IDX_MAX) {
765
-			if (!__test_and_set_bit(idx, sched->state.used))
766
-				goto done;
835
+			u64 mask = BIT_ULL(idx);
836
+
837
+			if (sched->state.used & mask)
838
+				continue;
839
+
840
+			sched->state.used |= mask;
841
+			goto done;
767842 		}
768843 	}
769844 
770845 	/* Grab the first unused counter starting with idx */
771846 	idx = sched->state.counter;
772847 	for_each_set_bit_from(idx, c->idxmsk, INTEL_PMC_IDX_FIXED) {
773
-		if (!__test_and_set_bit(idx, sched->state.used)) {
774
-			if (sched->state.nr_gp++ >= sched->max_gp)
775
-				return false;
848
+		u64 mask = BIT_ULL(idx);
776849 
777
-			goto done;
778
-		}
850
+		if (c->flags & PERF_X86_EVENT_PAIR)
851
+			mask |= mask << 1;
852
+
853
+		if (sched->state.used & mask)
854
+			continue;
855
+
856
+		if (sched->state.nr_gp++ >= sched->max_gp)
857
+			return false;
858
+
859
+		sched->state.used |= mask;
860
+		goto done;
779861 	}
780862 
781863 	return false;
....@@ -852,20 +934,42 @@
852934 int x86_schedule_events(struct cpu_hw_events *cpuc, int n, int *assign)
853935 {
854936 	struct event_constraint *c;
855
-	unsigned long used_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
856937 	struct perf_event *e;
857
-	int i, wmin, wmax, unsched = 0;
938
+	int n0, i, wmin, wmax, unsched = 0;
858939 	struct hw_perf_event *hwc;
940
+	u64 used_mask = 0;
859941 
860
-	bitmap_zero(used_mask, X86_PMC_IDX_MAX);
942
+	/*
943
+	 * Compute the number of events already present; see x86_pmu_add(),
944
+	 * validate_group() and x86_pmu_commit_txn(). For the former two
945
+	 * cpuc->n_events hasn't been updated yet, while for the latter
946
+	 * cpuc->n_txn contains the number of events added in the current
947
+	 * transaction.
948
+	 */
949
+	n0 = cpuc->n_events;
950
+	if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
951
+		n0 -= cpuc->n_txn;
861952 
862
-	if (x86_pmu.start_scheduling)
863
-		x86_pmu.start_scheduling(cpuc);
953
+	static_call_cond(x86_pmu_start_scheduling)(cpuc);
864954 
865955 	for (i = 0, wmin = X86_PMC_IDX_MAX, wmax = 0; i < n; i++) {
866
-		cpuc->event_constraint[i] = NULL;
867
-		c = x86_pmu.get_event_constraints(cpuc, i, cpuc->event_list[i]);
868
-		cpuc->event_constraint[i] = c;
956
+		c = cpuc->event_constraint[i];
957
+
958
+		/*
959
+		 * Previously scheduled events should have a cached constraint,
960
+		 * while new events should not have one.
961
+		 */
962
+		WARN_ON_ONCE((c && i >= n0) || (!c && i < n0));
963
+
964
+		/*
965
+		 * Request constraints for new events; or for those events that
966
+		 * have a dynamic constraint -- for those the constraint can
967
+		 * change due to external factors (sibling state, allow_tfa).
968
+		 */
969
+		if (!c || (c->flags & PERF_X86_EVENT_DYNAMIC)) {
970
+			c = static_call(x86_pmu_get_event_constraints)(cpuc, i, cpuc->event_list[i]);
971
+			cpuc->event_constraint[i] = c;
972
+		}
869973 
870974 		wmin = min(wmin, c->weight);
871975 		wmax = max(wmax, c->weight);
....@@ -875,6 +979,8 @@
875979 	 * fastpath, try to reuse previous register
876980 	 */
877981 	for (i = 0; i < n; i++) {
982
+		u64 mask;
983
+
878984 		hwc = &cpuc->event_list[i]->hw;
879985 		c = cpuc->event_constraint[i];
880986 
....@@ -886,11 +992,16 @@
886992 		if (!test_bit(hwc->idx, c->idxmsk))
887993 			break;
888994 
995
+		mask = BIT_ULL(hwc->idx);
996
+		if (is_counter_pair(hwc))
997
+			mask |= mask << 1;
998
+
889999 		/* not already used */
890
-		if (test_bit(hwc->idx, used_mask))
1000
+		if (used_mask & mask)
8911001 			break;
8921002 
893
-		__set_bit(hwc->idx, used_mask);
1003
+		used_mask |= mask;
1004
+
8941005 		if (assign)
8951006 			assign[i] = hwc->idx;
8961007 	}
....@@ -913,6 +1024,15 @@
9131024 		    READ_ONCE(cpuc->excl_cntrs->exclusive_present))
9141025 			gpmax /= 2;
9151026 
1027
+		/*
1028
+		 * Reduce the amount of available counters to allow fitting
1029
+		 * the extra Merge events needed by large increment events.
1030
+		 */
1031
+		if (x86_pmu.flags & PMU_FL_PAIR) {
1032
+			gpmax = x86_pmu.num_counters - cpuc->n_pair;
1033
+			WARN_ON(gpmax <= 0);
1034
+		}
1035
+
9161036 		unsched = perf_assign_events(cpuc->event_constraint, n, wmin,
9171037 					     wmax, gpmax, assign);
9181038 	}
....@@ -930,32 +1050,63 @@
9301050 	if (!unsched && assign) {
9311051 		for (i = 0; i < n; i++) {
9321052 			e = cpuc->event_list[i];
933
-			e->hw.flags |= PERF_X86_EVENT_COMMITTED;
934
-			if (x86_pmu.commit_scheduling)
935
-				x86_pmu.commit_scheduling(cpuc, i, assign[i]);
1053
+			static_call_cond(x86_pmu_commit_scheduling)(cpuc, i, assign[i]);
9361054 		}
9371055 	} else {
938
-		for (i = 0; i < n; i++) {
1056
+		for (i = n0; i < n; i++) {
9391057 			e = cpuc->event_list[i];
940
-			/*
941
-			 * do not put_constraint() on comitted events,
942
-			 * because they are good to go
943
-			 */
944
-			if ((e->hw.flags & PERF_X86_EVENT_COMMITTED))
945
-				continue;
9461058 
9471059 			/*
9481060 			 * release events that failed scheduling
9491061 			 */
950
-			if (x86_pmu.put_event_constraints)
951
-				x86_pmu.put_event_constraints(cpuc, e);
1062
+			static_call_cond(x86_pmu_put_event_constraints)(cpuc, e);
1063
+
1064
+			cpuc->event_constraint[i] = NULL;
9521065 		}
9531066 	}
9541067 
955
-	if (x86_pmu.stop_scheduling)
956
-		x86_pmu.stop_scheduling(cpuc);
1068
+	static_call_cond(x86_pmu_stop_scheduling)(cpuc);
9571069 
9581070 	return unsched ? -EINVAL : 0;
1071
+}
1072
+
1073
+static int add_nr_metric_event(struct cpu_hw_events *cpuc,
1074
+			       struct perf_event *event)
1075
+{
1076
+	if (is_metric_event(event)) {
1077
+		if (cpuc->n_metric == INTEL_TD_METRIC_NUM)
1078
+			return -EINVAL;
1079
+		cpuc->n_metric++;
1080
+		cpuc->n_txn_metric++;
1081
+	}
1082
+
1083
+	return 0;
1084
+}
1085
+
1086
+static void del_nr_metric_event(struct cpu_hw_events *cpuc,
1087
+				struct perf_event *event)
1088
+{
1089
+	if (is_metric_event(event))
1090
+		cpuc->n_metric--;
1091
+}
1092
+
1093
+static int collect_event(struct cpu_hw_events *cpuc, struct perf_event *event,
1094
+			 int max_count, int n)
1095
+{
1096
+
1097
+	if (x86_pmu.intel_cap.perf_metrics && add_nr_metric_event(cpuc, event))
1098
+		return -EINVAL;
1099
+
1100
+	if (n >= max_count + cpuc->n_metric)
1101
+		return -EINVAL;
1102
+
1103
+	cpuc->event_list[n] = event;
1104
+	if (is_counter_pair(&event->hw)) {
1105
+		cpuc->n_pair++;
1106
+		cpuc->n_txn_pair++;
1107
+	}
1108
+
1109
+	return 0;
9591110 }
9601111 
9611112 /*
....@@ -971,25 +1122,44 @@
9711122 
9721123 	/* current number of events already accepted */
9731124 	n = cpuc->n_events;
1125
+	if (!cpuc->n_events)
1126
+		cpuc->pebs_output = 0;
1127
+
1128
+	if (!cpuc->is_fake && leader->attr.precise_ip) {
1129
+		/*
1130
+		 * For PEBS->PT, if !aux_event, the group leader (PT) went
1131
+		 * away, the group was broken down and this singleton event
1132
+		 * can't schedule any more.
1133
+		 */
1134
+		if (is_pebs_pt(leader) && !leader->aux_event)
1135
+			return -EINVAL;
1136
+
1137
+		/*
1138
+		 * pebs_output: 0: no PEBS so far, 1: PT, 2: DS
1139
+		 */
1140
+		if (cpuc->pebs_output &&
1141
+		    cpuc->pebs_output != is_pebs_pt(leader) + 1)
1142
+			return -EINVAL;
1143
+
1144
+		cpuc->pebs_output = is_pebs_pt(leader) + 1;
1145
+	}
9741146 
9751147 	if (is_x86_event(leader)) {
976
-		if (n >= max_count)
1148
+		if (collect_event(cpuc, leader, max_count, n))
9771149 			return -EINVAL;
978
-		cpuc->event_list[n] = leader;
9791150 		n++;
9801151 	}
1152
+
9811153 	if (!dogrp)
9821154 		return n;
9831155 
9841156 	for_each_sibling_event(event, leader) {
985
-		if (!is_x86_event(event) ||
986
-		    event->state <= PERF_EVENT_STATE_OFF)
1157
+		if (!is_x86_event(event) || event->state <= PERF_EVENT_STATE_OFF)
9871158 			continue;
9881159 
989
-		if (n >= max_count)
1160
+		if (collect_event(cpuc, event, max_count, n))
9901161 			return -EINVAL;
9911162 
992
-		cpuc->event_list[n] = event;
9931163 		n++;
9941164 	}
9951165 	return n;
....@@ -999,23 +1169,58 @@
9991169 				struct cpu_hw_events *cpuc, int i)
10001170 {
10011171 	struct hw_perf_event *hwc = &event->hw;
1172
+	int idx;
10021173 
1003
-	hwc->idx = cpuc->assign[i];
1174
+	idx = hwc->idx = cpuc->assign[i];
10041175 	hwc->last_cpu = smp_processor_id();
10051176 	hwc->last_tag = ++cpuc->tags[i];
10061177 
1007
-	if (hwc->idx == INTEL_PMC_IDX_FIXED_BTS) {
1178
+	switch (hwc->idx) {
1179
+	case INTEL_PMC_IDX_FIXED_BTS:
1180
+	case INTEL_PMC_IDX_FIXED_VLBR:
10081181 		hwc->config_base = 0;
10091182 		hwc->event_base	= 0;
1010
-	} else if (hwc->idx >= INTEL_PMC_IDX_FIXED) {
1183
+		break;
1184
+
1185
+	case INTEL_PMC_IDX_METRIC_BASE ... INTEL_PMC_IDX_METRIC_END:
1186
+		/* All the metric events are mapped onto the fixed counter 3. */
1187
+		idx = INTEL_PMC_IDX_FIXED_SLOTS;
1188
+		/* fall through */
1189
+	case INTEL_PMC_IDX_FIXED ... INTEL_PMC_IDX_FIXED_BTS-1:
10111190 		hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
1012
-		hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 + (hwc->idx - INTEL_PMC_IDX_FIXED);
1013
-		hwc->event_base_rdpmc = (hwc->idx - INTEL_PMC_IDX_FIXED) | 1<<30;
1014
-	} else {
1191
+		hwc->event_base = MSR_ARCH_PERFMON_FIXED_CTR0 +
1192
+				(idx - INTEL_PMC_IDX_FIXED);
1193
+		hwc->event_base_rdpmc = (idx - INTEL_PMC_IDX_FIXED) |
1194
+					INTEL_PMC_FIXED_RDPMC_BASE;
1195
+		break;
1196
+
1197
+	default:
10151198 		hwc->config_base = x86_pmu_config_addr(hwc->idx);
10161199 		hwc->event_base  = x86_pmu_event_addr(hwc->idx);
10171200 		hwc->event_base_rdpmc = x86_pmu_rdpmc_index(hwc->idx);
1201
+		break;
10181202 	}
1203
+}
1204
+
1205
+/**
1206
+ * x86_perf_rdpmc_index - Return PMC counter used for event
1207
+ * @event: the perf_event to which the PMC counter was assigned
1208
+ *
1209
+ * The counter assigned to this performance event may change if interrupts
1210
+ * are enabled. This counter should thus never be used while interrupts are
1211
+ * enabled. Before this function is used to obtain the assigned counter the
1212
+ * event should be checked for validity using, for example,
1213
+ * perf_event_read_local(), within the same interrupt disabled section in
1214
+ * which this counter is planned to be used.
1215
+ *
1216
+ * Return: The index of the performance monitoring counter assigned to
1217
+ * @perf_event.
1218
+ */
1219
+int x86_perf_rdpmc_index(struct perf_event *event)
1220
+{
1221
+	lockdep_assert_irqs_disabled();
1222
+
1223
+	return event->hw.event_base_rdpmc;
10191224 }
10201225 
10211226 static inline int match_prev_assignment(struct hw_perf_event *hwc,
....@@ -1098,7 +1303,7 @@
10981303 	cpuc->enabled = 1;
10991304 	barrier();
11001305 
1101
-	x86_pmu.enable_all(added);
1306
+	static_call(x86_pmu_enable_all)(added);
11021307 }
11031308 
11041309 static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
....@@ -1114,8 +1319,12 @@
11141319 	s64 period = hwc->sample_period;
11151320 	int ret = 0, idx = hwc->idx;
11161321 
1117
-	if (idx == INTEL_PMC_IDX_FIXED_BTS)
1322
+	if (unlikely(!hwc->event_base))
11181323 		return 0;
1324
+
1325
+	if (unlikely(is_topdown_count(event)) &&
1326
+	    x86_pmu.set_topdown_event_period)
1327
+		return x86_pmu.set_topdown_event_period(event);
11191328 
11201329 	/*
11211330 	 * If we are way outside a reasonable range then just skip forward:
....@@ -1156,6 +1365,13 @@
11561365 	wrmsrl(hwc->event_base, (u64)(-left) & x86_pmu.cntval_mask);
11571366 
11581367 	/*
1368
+	 * Sign extend the Merge event counter's upper 16 bits since
1369
+	 * we currently declare a 48-bit counter width
1370
+	 */
1371
+	if (is_counter_pair(hwc))
1372
+		wrmsrl(x86_pmu_event_addr(idx + 1), 0xffff);
1373
+
1374
+	/*
11591375 	 * Due to erratum on certan cpu we need
11601376 	 * a second write to be sure the register
11611377 	 * is updated properly
....@@ -1181,7 +1397,7 @@
11811397  * Add a single event to the PMU.
11821398  *
11831399  * The event is added to the group of enabled events
1184
- * but only if it can be scehduled with existing events.
1400
+ * but only if it can be scheduled with existing events.
11851401  */
11861402 static int x86_pmu_add(struct perf_event *event, int flags)
11871403 {
....@@ -1212,7 +1428,7 @@
12121428 	if (cpuc->txn_flags & PERF_PMU_TXN_ADD)
12131429 		goto done_collect;
12141430 
1215
-	ret = x86_pmu.schedule_events(cpuc, n, assign);
1431
+	ret = static_call(x86_pmu_schedule_events)(cpuc, n, assign);
12161432 	if (ret)
12171433 		goto out;
12181434 	/*
....@@ -1230,13 +1446,11 @@
12301446 	cpuc->n_added += n - n0;
12311447 	cpuc->n_txn += n - n0;
12321448 
1233
-	if (x86_pmu.add) {
1234
-		/*
1235
-		 * This is before x86_pmu_enable() will call x86_pmu_start(),
1236
-		 * so we enable LBRs before an event needs them etc..
1237
-		 */
1238
-		x86_pmu.add(event);
1239
-	}
1449
+	/*
1450
+	 * This is before x86_pmu_enable() will call x86_pmu_start(),
1451
+	 * so we enable LBRs before an event needs them etc..
1452
+	 */
1453
+	static_call_cond(x86_pmu_add)(event);
12401454 
12411455 	ret = 0;
12421456 out:
....@@ -1264,7 +1478,7 @@
12641478 	cpuc->events[idx] = event;
12651479 	__set_bit(idx, cpuc->active_mask);
12661480 	__set_bit(idx, cpuc->running);
1267
-	x86_pmu.enable(event);
1481
+	static_call(x86_pmu_enable)(event);
12681482 	perf_event_update_userpage(event);
12691483 }
12701484 
....@@ -1334,7 +1548,7 @@
13341548 	struct hw_perf_event *hwc = &event->hw;
13351549 
13361550 	if (test_bit(hwc->idx, cpuc->active_mask)) {
1337
-		x86_pmu.disable(event);
1551
+		static_call(x86_pmu_disable)(event);
13381552 		__clear_bit(hwc->idx, cpuc->active_mask);
13391553 		cpuc->events[hwc->idx] = NULL;
13401554 		WARN_ON_ONCE(hwc->state & PERF_HES_STOPPED);
....@@ -1355,11 +1569,6 @@
13551569 {
13561570 	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
13571571 	int i;
1358
-
1359
-	/*
1360
-	 * event is descheduled
1361
-	 */
1362
-	event->hw.flags &= ~PERF_X86_EVENT_COMMITTED;
13631572 
13641573 	/*
13651574 	 * If we're called during a txn, we only need to undo x86_pmu.add.
....@@ -1389,26 +1598,27 @@
13891598 	if (i >= cpuc->n_events - cpuc->n_added)
13901599 		--cpuc->n_added;
13911600 
1392
-	if (x86_pmu.put_event_constraints)
1393
-		x86_pmu.put_event_constraints(cpuc, event);
1601
+	static_call_cond(x86_pmu_put_event_constraints)(cpuc, event);
13941602 
13951603 	/* Delete the array entry. */
13961604 	while (++i < cpuc->n_events) {
13971605 		cpuc->event_list[i-1] = cpuc->event_list[i];
13981606 		cpuc->event_constraint[i-1] = cpuc->event_constraint[i];
13991607 	}
1608
+	cpuc->event_constraint[i-1] = NULL;
14001609 	--cpuc->n_events;
1610
+	if (x86_pmu.intel_cap.perf_metrics)
1611
+		del_nr_metric_event(cpuc, event);
14011612 
14021613 	perf_event_update_userpage(event);
14031614 
14041615 do_del:
1405
-	if (x86_pmu.del) {
1406
-		/*
1407
-		 * This is after x86_pmu_stop(); so we disable LBRs after any
1408
-		 * event can need them etc..
1409
-		 */
1410
-		x86_pmu.del(event);
1411
-	}
1616
+
1617
+	/*
1618
+	 * This is after x86_pmu_stop(); so we disable LBRs after any
1619
+	 * event can need them etc..
1620
+	 */
1621
+	static_call_cond(x86_pmu_del)(event);
14121622 }
14131623 
14141624 int x86_pmu_handle_irq(struct pt_regs *regs)
....@@ -1486,7 +1696,7 @@
14861696 		return NMI_DONE;
14871697 
14881698 	start_clock = sched_clock();
1489
-	ret = x86_pmu.handle_irq(regs);
1699
+	ret = static_call(x86_pmu_handle_irq)(regs);
14901700 	finish_clock = sched_clock();
14911701 
14921702 	perf_sample_event_took(finish_clock - start_clock);
....@@ -1562,78 +1772,19 @@
15621772 
15631773 }
15641774 
1565
-static struct attribute_group x86_pmu_format_group = {
1775
+static struct attribute_group x86_pmu_format_group __ro_after_init = {
15661776 	.name = "format",
15671777 	.attrs = NULL,
15681778 };
15691779 
1570
-/*
1571
- * Remove all undefined events (x86_pmu.event_map(id) == 0)
1572
- * out of events_attr attributes.
1573
- */
1574
-static void __init filter_events(struct attribute **attrs)
1575
-{
1576
-	struct device_attribute *d;
1577
-	struct perf_pmu_events_attr *pmu_attr;
1578
-	int offset = 0;
1579
-	int i, j;
1580
-
1581
-	for (i = 0; attrs[i]; i++) {
1582
-		d = (struct device_attribute *)attrs[i];
1583
-		pmu_attr = container_of(d, struct perf_pmu_events_attr, attr);
1584
-		/* str trumps id */
1585
-		if (pmu_attr->event_str)
1586
-			continue;
1587
-		if (x86_pmu.event_map(i + offset))
1588
-			continue;
1589
-
1590
-		for (j = i; attrs[j]; j++)
1591
-			attrs[j] = attrs[j + 1];
1592
-
1593
-		/* Check the shifted attr. */
1594
-		i--;
1595
-
1596
-		/*
1597
-		 * event_map() is index based, the attrs array is organized
1598
-		 * by increasing event index. If we shift the events, then
1599
-		 * we need to compensate for the event_map(), otherwise
1600
-		 * we are looking up the wrong event in the map
1601
-		 */
1602
-		offset++;
1603
-	}
1604
-}
1605
-
1606
-/* Merge two pointer arrays */
1607
-__init struct attribute **merge_attr(struct attribute **a, struct attribute **b)
1608
-{
1609
-	struct attribute **new;
1610
-	int j, i;
1611
-
1612
-	for (j = 0; a[j]; j++)
1613
-		;
1614
-	for (i = 0; b[i]; i++)
1615
-		j++;
1616
-	j++;
1617
-
1618
-	new = kmalloc_array(j, sizeof(struct attribute *), GFP_KERNEL);
1619
-	if (!new)
1620
-		return NULL;
1621
-
1622
-	j = 0;
1623
-	for (i = 0; a[i]; i++)
1624
-		new[j++] = a[i];
1625
-	for (i = 0; b[i]; i++)
1626
-		new[j++] = b[i];
1627
-	new[j] = NULL;
1628
-
1629
-	return new;
1630
-}
1631
-
16321780 ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, char *page)
16331781 {
1634
-	struct perf_pmu_events_attr *pmu_attr = \
1782
+	struct perf_pmu_events_attr *pmu_attr =
16351783 		container_of(attr, struct perf_pmu_events_attr, attr);
1636
-	u64 config = x86_pmu.event_map(pmu_attr->id);
1784
+	u64 config = 0;
1785
+
1786
+	if (pmu_attr->id < x86_pmu.max_events)
1787
+		config = x86_pmu.event_map(pmu_attr->id);
16371788 
16381789 	/* string trumps id */
16391790 	if (pmu_attr->event_str)
....@@ -1693,9 +1844,27 @@
16931844 	NULL,
16941845 };
16951846 
1696
-static struct attribute_group x86_pmu_events_group = {
1847
+/*
1848
+ * Remove all undefined events (x86_pmu.event_map(id) == 0)
1849
+ * out of events_attr attributes.
1850
+ */
1851
+static umode_t
1852
+is_visible(struct kobject *kobj, struct attribute *attr, int idx)
1853
+{
1854
+	struct perf_pmu_events_attr *pmu_attr;
1855
+
1856
+	if (idx >= x86_pmu.max_events)
1857
+		return 0;
1858
+
1859
+	pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr.attr);
1860
+	/* str trumps id */
1861
+	return pmu_attr->event_str || x86_pmu.event_map(idx) ? attr->mode : 0;
1862
+}
1863
+
1864
+static struct attribute_group x86_pmu_events_group __ro_after_init = {
16971865 	.name = "events",
16981866 	.attrs = events_attr,
1867
+	.is_visible = is_visible,
16991868 };
17001869 
17011870 ssize_t x86_event_sysfs_show(char *page, u64 config, u64 event)
....@@ -1740,6 +1909,38 @@
17401909 static struct attribute_group x86_pmu_attr_group;
17411910 static struct attribute_group x86_pmu_caps_group;
17421911 
1912
+static void x86_pmu_static_call_update(void)
1913
+{
1914
+	static_call_update(x86_pmu_handle_irq, x86_pmu.handle_irq);
1915
+	static_call_update(x86_pmu_disable_all, x86_pmu.disable_all);
1916
+	static_call_update(x86_pmu_enable_all, x86_pmu.enable_all);
1917
+	static_call_update(x86_pmu_enable, x86_pmu.enable);
1918
+	static_call_update(x86_pmu_disable, x86_pmu.disable);
1919
+
1920
+	static_call_update(x86_pmu_add, x86_pmu.add);
1921
+	static_call_update(x86_pmu_del, x86_pmu.del);
1922
+	static_call_update(x86_pmu_read, x86_pmu.read);
1923
+
1924
+	static_call_update(x86_pmu_schedule_events, x86_pmu.schedule_events);
1925
+	static_call_update(x86_pmu_get_event_constraints, x86_pmu.get_event_constraints);
1926
+	static_call_update(x86_pmu_put_event_constraints, x86_pmu.put_event_constraints);
1927
+
1928
+	static_call_update(x86_pmu_start_scheduling, x86_pmu.start_scheduling);
1929
+	static_call_update(x86_pmu_commit_scheduling, x86_pmu.commit_scheduling);
1930
+	static_call_update(x86_pmu_stop_scheduling, x86_pmu.stop_scheduling);
1931
+
1932
+	static_call_update(x86_pmu_sched_task, x86_pmu.sched_task);
1933
+	static_call_update(x86_pmu_swap_task_ctx, x86_pmu.swap_task_ctx);
1934
+
1935
+	static_call_update(x86_pmu_drain_pebs, x86_pmu.drain_pebs);
1936
+	static_call_update(x86_pmu_pebs_aliases, x86_pmu.pebs_aliases);
1937
+}
1938
+
1939
+static void _x86_pmu_read(struct perf_event *event)
1940
+{
1941
+	x86_perf_event_update(event);
1942
+}
1943
+
17431944 static int __init init_hw_perf_events(void)
17441945 {
17451946 	struct x86_pmu_quirk *quirk;
....@@ -1753,6 +1954,14 @@
17531954 		break;
17541955 	case X86_VENDOR_AMD:
17551956 		err = amd_pmu_init();
1957
+		break;
1958
+	case X86_VENDOR_HYGON:
1959
+		err = amd_pmu_init();
1960
+		x86_pmu.name = "HYGON";
1961
+		break;
1962
+	case X86_VENDOR_ZHAOXIN:
1963
+	case X86_VENDOR_CENTAUR:
1964
+		err = zhaoxin_pmu_init();
17561965 		break;
17571966 	default:
17581967 		err = -ENOTSUPP;
....@@ -1787,37 +1996,10 @@
17871996 
17881997 	x86_pmu_format_group.attrs = x86_pmu.format_attrs;
17891998 
1790
-	if (x86_pmu.caps_attrs) {
1791
-		struct attribute **tmp;
1792
-
1793
-		tmp = merge_attr(x86_pmu_caps_group.attrs, x86_pmu.caps_attrs);
1794
-		if (!WARN_ON(!tmp))
1795
-			x86_pmu_caps_group.attrs = tmp;
1796
-	}
1797
-
1798
-	if (x86_pmu.event_attrs)
1799
-		x86_pmu_events_group.attrs = x86_pmu.event_attrs;
1800
-
18011999 	if (!x86_pmu.events_sysfs_show)
18022000 		x86_pmu_events_group.attrs = &empty_attrs;
1803
-	else
1804
-		filter_events(x86_pmu_events_group.attrs);
18052001 
1806
-	if (x86_pmu.cpu_events) {
1807
-		struct attribute **tmp;
1808
-
1809
-		tmp = merge_attr(x86_pmu_events_group.attrs, x86_pmu.cpu_events);
1810
-		if (!WARN_ON(!tmp))
1811
-			x86_pmu_events_group.attrs = tmp;
1812
-	}
1813
-
1814
-	if (x86_pmu.attrs) {
1815
-		struct attribute **tmp;
1816
-
1817
-		tmp = merge_attr(x86_pmu_attr_group.attrs, x86_pmu.attrs);
1818
-		if (!WARN_ON(!tmp))
1819
-			x86_pmu_attr_group.attrs = tmp;
1820
-	}
2002
+	pmu.attr_update = x86_pmu.attr_update;
18212003 
18222004 	pr_info("... version:                %d\n",     x86_pmu.version);
18232005 	pr_info("... bit width:              %d\n",     x86_pmu.cntval_bits);
....@@ -1826,6 +2008,11 @@
18262008 	pr_info("... max period:             %016Lx\n", x86_pmu.max_period);
18272009 	pr_info("... fixed-purpose events:   %d\n",     x86_pmu.num_counters_fixed);
18282010 	pr_info("... event mask:             %016Lx\n", x86_pmu.intel_ctrl);
2011
+
2012
+	if (!x86_pmu.read)
2013
+		x86_pmu.read = _x86_pmu_read;
2014
+
2015
+	x86_pmu_static_call_update();
18292016 
18302017 	/*
18312018 	 * Install callbacks. Core will call them for each online
....@@ -1863,11 +2050,9 @@
18632050 }
18642051 early_initcall(init_hw_perf_events);
18652052 
1866
-static inline void x86_pmu_read(struct perf_event *event)
2053
+static void x86_pmu_read(struct perf_event *event)
18672054 {
1868
-	if (x86_pmu.read)
1869
-		return x86_pmu.read(event);
1870
-	x86_perf_event_update(event);
2055
+	static_call(x86_pmu_read)(event);
18712056 }
18722057 
18732058 /*
....@@ -1891,6 +2076,8 @@
18912076 
18922077 	perf_pmu_disable(pmu);
18932078 	__this_cpu_write(cpu_hw_events.n_txn, 0);
2079
+	__this_cpu_write(cpu_hw_events.n_txn_pair, 0);
2080
+	__this_cpu_write(cpu_hw_events.n_txn_metric, 0);
18942081 }
18952082 
18962083 /*
....@@ -1916,6 +2103,8 @@
19162103 	 */
19172104 	__this_cpu_sub(cpu_hw_events.n_added, __this_cpu_read(cpu_hw_events.n_txn));
19182105 	__this_cpu_sub(cpu_hw_events.n_events, __this_cpu_read(cpu_hw_events.n_txn));
2106
+	__this_cpu_sub(cpu_hw_events.n_pair, __this_cpu_read(cpu_hw_events.n_txn_pair));
2107
+	__this_cpu_sub(cpu_hw_events.n_metric, __this_cpu_read(cpu_hw_events.n_txn_metric));
19192108 	perf_pmu_enable(pmu);
19202109 }
19212110 
....@@ -1944,7 +2133,7 @@
19442133 	if (!x86_pmu_initialized())
19452134 		return -EAGAIN;
19462135 
1947
-	ret = x86_pmu.schedule_events(cpuc, n, assign);
2136
+	ret = static_call(x86_pmu_schedule_events)(cpuc, n, assign);
19482137 	if (ret)
19492138 		return ret;
19502139 
....@@ -2004,7 +2193,7 @@
20042193 	if (IS_ERR(fake_cpuc))
20052194 		return PTR_ERR(fake_cpuc);
20062195 
2007
-	c = x86_pmu.get_event_constraints(fake_cpuc, -1, event);
2196
+	c = x86_pmu.get_event_constraints(fake_cpuc, 0, event);
20082197 
20092198 	if (!c || !c->weight)
20102199 		ret = -EINVAL;
....@@ -2052,8 +2241,7 @@
20522241 	if (n < 0)
20532242 		goto out;
20542243 
2055
-	fake_cpuc->n_events = n;
2056
-
2244
+	fake_cpuc->n_events = 0;
20572245 	ret = x86_pmu.schedule_events(fake_cpuc, n, NULL);
20582246 
20592247 out:
....@@ -2106,11 +2294,6 @@
21062294 	return err;
21072295 }
21082296 
2109
-static void refresh_pce(void *ignored)
2110
-{
2111
-	load_mm_cr4(this_cpu_read(cpu_tlbstate.loaded_mm));
2112
-}
2113
-
21142297 static void x86_pmu_event_mapped(struct perf_event *event, struct mm_struct *mm)
21152298 {
21162299 	if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
....@@ -2123,13 +2306,13 @@
21232306 	 * userspace with CR4.PCE clear while another task is still
21242307 	 * doing on_each_cpu_mask() to propagate CR4.PCE.
21252308 	 *
2126
-	 * For now, this can't happen because all callers hold mmap_sem
2309
+	 * For now, this can't happen because all callers hold mmap_lock
21272310 	 * for write.  If this changes, we'll need a different solution.
21282311 	 */
2129
-	lockdep_assert_held_exclusive(&mm->mmap_sem);
2312
+	mmap_assert_write_locked(mm);
21302313 
21312314 	if (atomic_inc_return(&mm->context.perf_rdpmc_allowed) == 1)
2132
-		on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1);
2315
+		on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1);
21332316 }
21342317 
21352318 static void x86_pmu_event_unmapped(struct perf_event *event, struct mm_struct *mm)
....@@ -2139,22 +2322,20 @@
21392322 		return;
21402323 
21412324 	if (atomic_dec_and_test(&mm->context.perf_rdpmc_allowed))
2142
-		on_each_cpu_mask(mm_cpumask(mm), refresh_pce, NULL, 1);
2325
+		on_each_cpu_mask(mm_cpumask(mm), cr4_update_pce, NULL, 1);
21432326 }
21442327 
21452328 static int x86_pmu_event_idx(struct perf_event *event)
21462329 {
2147
-	int idx = event->hw.idx;
2330
+	struct hw_perf_event *hwc = &event->hw;
21482331 
2149
-	if (!(event->hw.flags & PERF_X86_EVENT_RDPMC_ALLOWED))
2332
+	if (!(hwc->flags & PERF_X86_EVENT_RDPMC_ALLOWED))
21502333 		return 0;
21512334 
2152
-	if (x86_pmu.num_counters_fixed && idx >= INTEL_PMC_IDX_FIXED) {
2153
-		idx -= INTEL_PMC_IDX_FIXED;
2154
-		idx |= 1 << 30;
2155
-	}
2156
-
2157
-	return idx + 1;
2335
+	if (is_metric_idx(hwc->idx))
2336
+		return INTEL_PMC_FIXED_RDPMC_METRICS + 1;
2337
+	else
2338
+		return hwc->event_base_rdpmc + 1;
21582339 }
21592340 
21602341 static ssize_t get_attr_rdpmc(struct device *cdev,
....@@ -2181,20 +2362,25 @@
21812362 	if (x86_pmu.attr_rdpmc_broken)
21822363 		return -ENOTSUPP;
21832364 
2184
-	if ((val == 2) != (x86_pmu.attr_rdpmc == 2)) {
2365
+	if (val != x86_pmu.attr_rdpmc) {
21852366 		/*
2186
-		 * Changing into or out of always available, aka
2187
-		 * perf-event-bypassing mode.  This path is extremely slow,
2367
+		 * Changing into or out of never available or always available,
2368
+		 * aka perf-event-bypassing mode. This path is extremely slow,
21882369 		 * but only root can trigger it, so it's okay.
21892370 		 */
2371
+		if (val == 0)
2372
+			static_branch_inc(&rdpmc_never_available_key);
2373
+		else if (x86_pmu.attr_rdpmc == 0)
2374
+			static_branch_dec(&rdpmc_never_available_key);
2375
+
21902376 		if (val == 2)
21912377 			static_branch_inc(&rdpmc_always_available_key);
2192
-		else
2378
+		else if (x86_pmu.attr_rdpmc == 2)
21932379 			static_branch_dec(&rdpmc_always_available_key);
2194
-		on_each_cpu(refresh_pce, NULL, 1);
2195
-	}
21962380 
2197
-	x86_pmu.attr_rdpmc = val;
2381
+		on_each_cpu(cr4_update_pce, NULL, 1);
2382
+		x86_pmu.attr_rdpmc = val;
2383
+	}
21982384 
21992385 	return count;
22002386 }
....@@ -2206,7 +2392,7 @@
22062392 	NULL,
22072393 };
22082394 
2209
-static struct attribute_group x86_pmu_attr_group = {
2395
+static struct attribute_group x86_pmu_attr_group __ro_after_init = {
22102396 	.attrs = x86_pmu_attrs,
22112397 };
22122398 
....@@ -2224,7 +2410,7 @@
22242410 	NULL
22252411 };
22262412 
2227
-static struct attribute_group x86_pmu_caps_group = {
2413
+static struct attribute_group x86_pmu_caps_group __ro_after_init = {
22282414 	.name = "caps",
22292415 	.attrs = x86_pmu_caps_attrs,
22302416 };
....@@ -2239,8 +2425,13 @@
22392425 
22402426 static void x86_pmu_sched_task(struct perf_event_context *ctx, bool sched_in)
22412427 {
2242
-	if (x86_pmu.sched_task)
2243
-		x86_pmu.sched_task(ctx, sched_in);
2428
+	static_call_cond(x86_pmu_sched_task)(ctx, sched_in);
2429
+}
2430
+
2431
+static void x86_pmu_swap_task_ctx(struct perf_event_context *prev,
2432
+				  struct perf_event_context *next)
2433
+{
2434
+	static_call_cond(x86_pmu_swap_task_ctx)(prev, next);
22442435 }
22452436 
22462437 void perf_check_microcode(void)
....@@ -2258,6 +2449,17 @@
22582449 		if (x86_pmu.limit_period(event, value) > value)
22592450 			return -EINVAL;
22602451 	}
2452
+
2453
+	return 0;
2454
+}
2455
+
2456
+static int x86_pmu_aux_output_match(struct perf_event *event)
2457
+{
2458
+	if (!(pmu.capabilities & PERF_PMU_CAP_AUX_OUTPUT))
2459
+		return 0;
2460
+
2461
+	if (x86_pmu.aux_output_match)
2462
+		return x86_pmu.aux_output_match(event);
22612463 
22622464 	return 0;
22632465 }
....@@ -2285,8 +2487,10 @@
22852487 
22862488 	.event_idx		= x86_pmu_event_idx,
22872489 	.sched_task		= x86_pmu_sched_task,
2288
-	.task_ctx_size          = sizeof(struct x86_perf_task_context),
2490
+	.swap_task_ctx		= x86_pmu_swap_task_ctx,
22892491 	.check_period		= x86_pmu_check_period,
2492
+
2493
+	.aux_output_match	= x86_pmu_aux_output_match,
22902494 };
22912495 
22922496 void arch_perf_update_userpage(struct perf_event *event,
....@@ -2329,13 +2533,23 @@
23292533 	cyc2ns_read_end();
23302534 }
23312535 
2536
+/*
2537
+ * Determine whether the regs were taken from an irq/exception handler rather
2538
+ * than from perf_arch_fetch_caller_regs().
2539
+ */
2540
+static bool perf_hw_regs(struct pt_regs *regs)
2541
+{
2542
+	return regs->flags & X86_EFLAGS_FIXED;
2543
+}
2544
+
23322545 void
23332546 perf_callchain_kernel(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
23342547 {
2548
+	struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
23352549 	struct unwind_state state;
23362550 	unsigned long addr;
23372551 
2338
-	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2552
+	if (guest_cbs && guest_cbs->is_in_guest()) {
23392553 		/* TODO: We don't support guest os callchain now */
23402554 		return;
23412555 	}
....@@ -2343,8 +2557,12 @@
23432557 	if (perf_callchain_store(entry, regs->ip))
23442558 		return;
23452559 
2346
-	for (unwind_start(&state, current, regs, NULL); !unwind_done(&state);
2347
-	     unwind_next_frame(&state)) {
2560
+	if (perf_hw_regs(regs))
2561
+		unwind_start(&state, current, regs, NULL);
2562
+	else
2563
+		unwind_start(&state, current, NULL, (void *)regs->sp);
2564
+
2565
+	for (; !unwind_done(&state); unwind_next_frame(&state)) {
23482566 		addr = unwind_get_return_address(&state);
23492567 		if (!addr || perf_callchain_store(entry, addr))
23502568 			return;
....@@ -2395,7 +2613,7 @@
23952613 	/* 32-bit process in 64-bit kernel. */
23962614 	unsigned long ss_base, cs_base;
23972615 	struct stack_frame_ia32 frame;
2398
-	const void __user *fp;
2616
+	const struct stack_frame_ia32 __user *fp;
23992617 
24002618 	if (!test_thread_flag(TIF_IA32))
24012619 		return 0;
....@@ -2406,18 +2624,12 @@
24062624 	fp = compat_ptr(ss_base + regs->bp);
24072625 	pagefault_disable();
24082626 	while (entry->nr < entry->max_stack) {
2409
-		unsigned long bytes;
2410
-		frame.next_frame     = 0;
2411
-		frame.return_address = 0;
2412
-
24132627 		if (!valid_user_frame(fp, sizeof(frame)))
24142628 			break;
24152629 
2416
-		bytes = __copy_from_user_nmi(&frame.next_frame, fp, 4);
2417
-		if (bytes != 0)
2630
+		if (__get_user(frame.next_frame, &fp->next_frame))
24182631 			break;
2419
-		bytes = __copy_from_user_nmi(&frame.return_address, fp+4, 4);
2420
-		if (bytes != 0)
2632
+		if (__get_user(frame.return_address, &fp->return_address))
24212633 			break;
24222634 
24232635 		perf_callchain_store(entry, cs_base + frame.return_address);
....@@ -2437,10 +2649,11 @@
24372649 void
24382650 perf_callchain_user(struct perf_callchain_entry_ctx *entry, struct pt_regs *regs)
24392651 {
2652
+	struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
24402653 	struct stack_frame frame;
2441
-	const unsigned long __user *fp;
2654
+	const struct stack_frame __user *fp;
24422655 
2443
-	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2656
+	if (guest_cbs && guest_cbs->is_in_guest()) {
24442657 		/* TODO: We don't support guest os callchain now */
24452658 		return;
24462659 	}
....@@ -2451,7 +2664,7 @@
24512664 	if (regs->flags & (X86_VM_MASK | PERF_EFLAGS_VM))
24522665 		return;
24532666 
2454
-	fp = (unsigned long __user *)regs->bp;
2667
+	fp = (void __user *)regs->bp;
24552668 
24562669 	perf_callchain_store(entry, regs->ip);
24572670 
....@@ -2463,19 +2676,12 @@
24632676 
24642677 	pagefault_disable();
24652678 	while (entry->nr < entry->max_stack) {
2466
-		unsigned long bytes;
2467
-
2468
-		frame.next_frame	     = NULL;
2469
-		frame.return_address = 0;
2470
-
24712679 		if (!valid_user_frame(fp, sizeof(frame)))
24722680 			break;
24732681 
2474
-		bytes = __copy_from_user_nmi(&frame.next_frame, fp, sizeof(*fp));
2475
-		if (bytes != 0)
2682
+		if (__get_user(frame.next_frame, &fp->next_frame))
24762683 			break;
2477
-		bytes = __copy_from_user_nmi(&frame.return_address, fp + 1, sizeof(*fp));
2478
-		if (bytes != 0)
2684
+		if (__get_user(frame.return_address, &fp->return_address))
24792685 			break;
24802686 
24812687 		perf_callchain_store(entry, frame.return_address);
....@@ -2524,18 +2730,21 @@
25242730 
25252731 unsigned long perf_instruction_pointer(struct pt_regs *regs)
25262732 {
2527
-	if (perf_guest_cbs && perf_guest_cbs->is_in_guest())
2528
-		return perf_guest_cbs->get_guest_ip();
2733
+	struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
2734
+
2735
+	if (guest_cbs && guest_cbs->is_in_guest())
2736
+		return guest_cbs->get_guest_ip();
25292737 
25302738 	return regs->ip + code_segment_base(regs);
25312739 }
25322740 
25332741 unsigned long perf_misc_flags(struct pt_regs *regs)
25342742 {
2743
+	struct perf_guest_info_callbacks *guest_cbs = perf_get_guest_cbs();
25352744 	int misc = 0;
25362745 
2537
-	if (perf_guest_cbs && perf_guest_cbs->is_in_guest()) {
2538
-		if (perf_guest_cbs->is_user_mode())
2746
+	if (guest_cbs && guest_cbs->is_in_guest()) {
2747
+		if (guest_cbs->is_user_mode())
25392748 			misc |= PERF_RECORD_MISC_GUEST_USER;
25402749 		else
25412750 			misc |= PERF_RECORD_MISC_GUEST_KERNEL;