forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-09 b22da3d8526a935aa31e086e63f60ff3246cb61c 
 kernel/arch/s390/kernel/perf_cpum_cf.c
....@@ -2,81 +2,19 @@
22 /*
33  * Performance event support for s390x - CPU-measurement Counter Facility
44  *
5
- *  Copyright IBM Corp. 2012, 2017
6
- *  Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com>
5
+ *  Copyright IBM Corp. 2012, 2019
6
+ *  Author(s): Hendrik Brueckner <brueckner@linux.ibm.com>
77  */
88 #define KMSG_COMPONENT	"cpum_cf"
99 #define pr_fmt(fmt)	KMSG_COMPONENT ": " fmt
1010 
1111 #include <linux/kernel.h>
1212 #include <linux/kernel_stat.h>
13
-#include <linux/perf_event.h>
1413 #include <linux/percpu.h>
1514 #include <linux/notifier.h>
1615 #include <linux/init.h>
1716 #include <linux/export.h>
18
-#include <asm/ctl_reg.h>
19
-#include <asm/irq.h>
20
-#include <asm/cpu_mf.h>
21
-
22
-enum cpumf_ctr_set {
23
-	CPUMF_CTR_SET_BASIC   = 0,    /* Basic Counter Set */
24
-	CPUMF_CTR_SET_USER    = 1,    /* Problem-State Counter Set */
25
-	CPUMF_CTR_SET_CRYPTO  = 2,    /* Crypto-Activity Counter Set */
26
-	CPUMF_CTR_SET_EXT     = 3,    /* Extended Counter Set */
27
-	CPUMF_CTR_SET_MT_DIAG = 4,    /* MT-diagnostic Counter Set */
28
-
29
-	/* Maximum number of counter sets */
30
-	CPUMF_CTR_SET_MAX,
31
-};
32
-
33
-#define CPUMF_LCCTL_ENABLE_SHIFT    16
34
-#define CPUMF_LCCTL_ACTCTL_SHIFT     0
35
-static const u64 cpumf_state_ctl[CPUMF_CTR_SET_MAX] = {
36
-	[CPUMF_CTR_SET_BASIC]	= 0x02,
37
-	[CPUMF_CTR_SET_USER]	= 0x04,
38
-	[CPUMF_CTR_SET_CRYPTO]	= 0x08,
39
-	[CPUMF_CTR_SET_EXT]	= 0x01,
40
-	[CPUMF_CTR_SET_MT_DIAG] = 0x20,
41
-};
42
-
43
-static void ctr_set_enable(u64 *state, int ctr_set)
44
-{
45
-	*state |= cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ENABLE_SHIFT;
46
-}
47
-static void ctr_set_disable(u64 *state, int ctr_set)
48
-{
49
-	*state &= ~(cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ENABLE_SHIFT);
50
-}
51
-static void ctr_set_start(u64 *state, int ctr_set)
52
-{
53
-	*state |= cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ACTCTL_SHIFT;
54
-}
55
-static void ctr_set_stop(u64 *state, int ctr_set)
56
-{
57
-	*state &= ~(cpumf_state_ctl[ctr_set] << CPUMF_LCCTL_ACTCTL_SHIFT);
58
-}
59
-
60
-/* Local CPUMF event structure */
61
-struct cpu_hw_events {
62
-	struct cpumf_ctr_info	info;
63
-	atomic_t		ctr_set[CPUMF_CTR_SET_MAX];
64
-	u64			state, tx_state;
65
-	unsigned int		flags;
66
-	unsigned int		txn_flags;
67
-};
68
-static DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = {
69
-	.ctr_set = {
70
-		[CPUMF_CTR_SET_BASIC]	= ATOMIC_INIT(0),
71
-		[CPUMF_CTR_SET_USER]	= ATOMIC_INIT(0),
72
-		[CPUMF_CTR_SET_CRYPTO]	= ATOMIC_INIT(0),
73
-		[CPUMF_CTR_SET_EXT]	= ATOMIC_INIT(0),
74
-		[CPUMF_CTR_SET_MT_DIAG] = ATOMIC_INIT(0),
75
-	},
76
-	.state = 0,
77
-	.flags = 0,
78
-	.txn_flags = 0,
79
-};
17
+#include <asm/cpu_mcf.h>
8018 
8119 static enum cpumf_ctr_set get_counter_set(u64 event)
8220 {
....@@ -88,7 +26,7 @@
8826 		set = CPUMF_CTR_SET_USER;
8927 	else if (event < 128)
9028 		set = CPUMF_CTR_SET_CRYPTO;
91
-	else if (event < 256)
29
+	else if (event < 288)
9230 		set = CPUMF_CTR_SET_EXT;
9331 	else if (event >= 448 && event < 496)
9432 		set = CPUMF_CTR_SET_MT_DIAG;
....@@ -98,11 +36,11 @@
9836 
9937 static int validate_ctr_version(const struct hw_perf_event *hwc)
10038 {
101
-	struct cpu_hw_events *cpuhw;
39
+	struct cpu_cf_events *cpuhw;
10240 	int err = 0;
10341 	u16 mtdiag_ctl;
10442 
105
-	cpuhw = &get_cpu_var(cpu_hw_events);
43
+	cpuhw = &get_cpu_var(cpu_cf_events);
10644 
10745 	/* check required version for counter sets */
10846 	switch (hwc->config_base) {
....@@ -112,12 +50,19 @@
11250 			err = -EOPNOTSUPP;
11351 		break;
11452 	case CPUMF_CTR_SET_CRYPTO:
53
+		if ((cpuhw->info.csvn >= 1 && cpuhw->info.csvn <= 5 &&
54
+		     hwc->config > 79) ||
55
+		    (cpuhw->info.csvn >= 6 && hwc->config > 83))
56
+			err = -EOPNOTSUPP;
57
+		break;
11558 	case CPUMF_CTR_SET_EXT:
11659 		if (cpuhw->info.csvn < 1)
11760 			err = -EOPNOTSUPP;
11861 		if ((cpuhw->info.csvn == 1 && hwc->config > 159) ||
11962 		    (cpuhw->info.csvn == 2 && hwc->config > 175) ||
120
-		    (cpuhw->info.csvn  > 2 && hwc->config > 255))
63
+		    (cpuhw->info.csvn >= 3 && cpuhw->info.csvn <= 5
64
+		     && hwc->config > 255) ||
65
+		    (cpuhw->info.csvn >= 6 && hwc->config > 287))
12166 			err = -EOPNOTSUPP;
12267 		break;
12368 	case CPUMF_CTR_SET_MT_DIAG:
....@@ -135,7 +80,7 @@
13580 		 * Thus, the counters can only be used if SMT is on and the
13681 		 * counter set is enabled and active.
13782 		 */
138
-		mtdiag_ctl = cpumf_state_ctl[CPUMF_CTR_SET_MT_DIAG];
83
+		mtdiag_ctl = cpumf_ctr_ctl[CPUMF_CTR_SET_MT_DIAG];
13984 		if (!((cpuhw->info.auth_ctl & mtdiag_ctl) &&
14085 		      (cpuhw->info.enable_ctl & mtdiag_ctl) &&
14186 		      (cpuhw->info.act_ctl & mtdiag_ctl)))
....@@ -143,28 +88,28 @@
14388 		break;
14489 	}
14590 
146
-	put_cpu_var(cpu_hw_events);
91
+	put_cpu_var(cpu_cf_events);
14792 	return err;
14893 }
14994 
15095 static int validate_ctr_auth(const struct hw_perf_event *hwc)
15196 {
152
-	struct cpu_hw_events *cpuhw;
97
+	struct cpu_cf_events *cpuhw;
15398 	u64 ctrs_state;
15499 	int err = 0;
155100 
156
-	cpuhw = &get_cpu_var(cpu_hw_events);
101
+	cpuhw = &get_cpu_var(cpu_cf_events);
157102 
158103 	/* Check authorization for cpu counter sets.
159104 	 * If the particular CPU counter set is not authorized,
160105 	 * return with -ENOENT in order to fall back to other
161106 	 * PMUs that might suffice the event request.
162107 	 */
163
-	ctrs_state = cpumf_state_ctl[hwc->config_base];
108
+	ctrs_state = cpumf_ctr_ctl[hwc->config_base];
164109 	if (!(ctrs_state & cpuhw->info.auth_ctl))
165110 		err = -ENOENT;
166111 
167
-	put_cpu_var(cpu_hw_events);
112
+	put_cpu_var(cpu_cf_events);
168113 	return err;
169114 }
170115 
....@@ -175,7 +120,7 @@
175120  */
176121 static void cpumf_pmu_enable(struct pmu *pmu)
177122 {
178
-	struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
123
+	struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
179124 	int err;
180125 
181126 	if (cpuhw->flags & PMU_F_ENABLED)
....@@ -198,7 +143,7 @@
198143  */
199144 static void cpumf_pmu_disable(struct pmu *pmu)
200145 {
201
-	struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
146
+	struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
202147 	int err;
203148 	u64 inactive;
204149 
....@@ -222,86 +167,13 @@
222167 /* Used to avoid races in calling reserve/release_cpumf_hardware */
223168 static DEFINE_MUTEX(pmc_reserve_mutex);
224169 
225
-/* CPU-measurement alerts for the counter facility */
226
-static void cpumf_measurement_alert(struct ext_code ext_code,
227
-				    unsigned int alert, unsigned long unused)
228
-{
229
-	struct cpu_hw_events *cpuhw;
230
-
231
-	if (!(alert & CPU_MF_INT_CF_MASK))
232
-		return;
233
-
234
-	inc_irq_stat(IRQEXT_CMC);
235
-	cpuhw = this_cpu_ptr(&cpu_hw_events);
236
-
237
-	/* Measurement alerts are shared and might happen when the PMU
238
-	 * is not reserved.  Ignore these alerts in this case. */
239
-	if (!(cpuhw->flags & PMU_F_RESERVED))
240
-		return;
241
-
242
-	/* counter authorization change alert */
243
-	if (alert & CPU_MF_INT_CF_CACA)
244
-		qctri(&cpuhw->info);
245
-
246
-	/* loss of counter data alert */
247
-	if (alert & CPU_MF_INT_CF_LCDA)
248
-		pr_err("CPU[%i] Counter data was lost\n", smp_processor_id());
249
-
250
-	/* loss of MT counter data alert */
251
-	if (alert & CPU_MF_INT_CF_MTDA)
252
-		pr_warn("CPU[%i] MT counter data was lost\n",
253
-			smp_processor_id());
254
-}
255
-
256
-#define PMC_INIT      0
257
-#define PMC_RELEASE   1
258
-static void setup_pmc_cpu(void *flags)
259
-{
260
-	struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
261
-
262
-	switch (*((int *) flags)) {
263
-	case PMC_INIT:
264
-		memset(&cpuhw->info, 0, sizeof(cpuhw->info));
265
-		qctri(&cpuhw->info);
266
-		cpuhw->flags |= PMU_F_RESERVED;
267
-		break;
268
-
269
-	case PMC_RELEASE:
270
-		cpuhw->flags &= ~PMU_F_RESERVED;
271
-		break;
272
-	}
273
-
274
-	/* Disable CPU counter sets */
275
-	lcctl(0);
276
-}
277
-
278
-/* Initialize the CPU-measurement facility */
279
-static int reserve_pmc_hardware(void)
280
-{
281
-	int flags = PMC_INIT;
282
-
283
-	on_each_cpu(setup_pmc_cpu, &flags, 1);
284
-	irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT);
285
-
286
-	return 0;
287
-}
288
-
289
-/* Release the CPU-measurement facility */
290
-static void release_pmc_hardware(void)
291
-{
292
-	int flags = PMC_RELEASE;
293
-
294
-	on_each_cpu(setup_pmc_cpu, &flags, 1);
295
-	irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT);
296
-}
297
-
298170 /* Release the PMU if event is the last perf event */
299171 static void hw_perf_event_destroy(struct perf_event *event)
300172 {
301173 	if (!atomic_add_unless(&num_events, -1, 1)) {
302174 		mutex_lock(&pmc_reserve_mutex);
303175 		if (atomic_dec_return(&num_events) == 0)
304
-			release_pmc_hardware();
176
+			__kernel_cpumcf_end();
305177 		mutex_unlock(&pmc_reserve_mutex);
306178 	}
307179 }
....@@ -327,15 +199,15 @@
327199 	[PERF_COUNT_HW_BUS_CYCLES]	    = -1,
328200 };
329201 
330
-static int __hw_perf_event_init(struct perf_event *event)
202
+static int __hw_perf_event_init(struct perf_event *event, unsigned int type)
331203 {
332204 	struct perf_event_attr *attr = &event->attr;
333205 	struct hw_perf_event *hwc = &event->hw;
334206 	enum cpumf_ctr_set set;
335
-	int err;
207
+	int err = 0;
336208 	u64 ev;
337209 
338
-	switch (attr->type) {
210
+	switch (type) {
339211 	case PERF_TYPE_RAW:
340212 		/* Raw events are used to access counters directly,
341213 		 * hence do not permit excludes */
....@@ -402,12 +274,14 @@
402274 	/* Initialize for using the CPU-measurement counter facility */
403275 	if (!atomic_inc_not_zero(&num_events)) {
404276 		mutex_lock(&pmc_reserve_mutex);
405
-		if (atomic_read(&num_events) == 0 && reserve_pmc_hardware())
277
+		if (atomic_read(&num_events) == 0 && __kernel_cpumcf_begin())
406278 			err = -EBUSY;
407279 		else
408280 			atomic_inc(&num_events);
409281 		mutex_unlock(&pmc_reserve_mutex);
410282 	}
283
+	if (err)
284
+		return err;
411285 	event->destroy = hw_perf_event_destroy;
412286 
413287 	/* Finally, validate version and authorization of the counter set */
....@@ -418,19 +292,38 @@
418292 	return err;
419293 }
420294 
295
+/* Events CPU_CYLCES and INSTRUCTIONS can be submitted with two different
296
+ * attribute::type values:
297
+ * - PERF_TYPE_HARDWARE:
298
+ * - pmu->type:
299
+ * Handle both type of invocations identical. They address the same hardware.
300
+ * The result is different when event modifiers exclude_kernel and/or
301
+ * exclude_user are also set.
302
+ */
303
+static int cpumf_pmu_event_type(struct perf_event *event)
304
+{
305
+	u64 ev = event->attr.config;
306
+
307
+	if (cpumf_generic_events_basic[PERF_COUNT_HW_CPU_CYCLES] == ev ||
308
+	    cpumf_generic_events_basic[PERF_COUNT_HW_INSTRUCTIONS] == ev ||
309
+	    cpumf_generic_events_user[PERF_COUNT_HW_CPU_CYCLES] == ev ||
310
+	    cpumf_generic_events_user[PERF_COUNT_HW_INSTRUCTIONS] == ev)
311
+		return PERF_TYPE_HARDWARE;
312
+	return PERF_TYPE_RAW;
313
+}
314
+
421315 static int cpumf_pmu_event_init(struct perf_event *event)
422316 {
317
+	unsigned int type = event->attr.type;
423318 	int err;
424319 
425
-	switch (event->attr.type) {
426
-	case PERF_TYPE_HARDWARE:
427
-	case PERF_TYPE_HW_CACHE:
428
-	case PERF_TYPE_RAW:
429
-		err = __hw_perf_event_init(event);
430
-		break;
431
-	default:
320
+	if (type == PERF_TYPE_HARDWARE || type == PERF_TYPE_RAW)
321
+		err = __hw_perf_event_init(event, type);
322
+	else if (event->pmu->type == type)
323
+		/* Registered as unknown PMU */
324
+		err = __hw_perf_event_init(event, cpumf_pmu_event_type(event));
325
+	else
432326 		return -ENOENT;
433
-	}
434327 
435328 	if (unlikely(err) && event->destroy)
436329 		event->destroy(event);
....@@ -488,7 +381,7 @@
488381 
489382 static void cpumf_pmu_start(struct perf_event *event, int flags)
490383 {
491
-	struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
384
+	struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
492385 	struct hw_perf_event *hwc = &event->hw;
493386 
494387 	if (WARN_ON_ONCE(!(hwc->state & PERF_HES_STOPPED)))
....@@ -519,7 +412,7 @@
519412 
520413 static void cpumf_pmu_stop(struct perf_event *event, int flags)
521414 {
522
-	struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
415
+	struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
523416 	struct hw_perf_event *hwc = &event->hw;
524417 
525418 	if (!(hwc->state & PERF_HES_STOPPED)) {
....@@ -540,7 +433,7 @@
540433 
541434 static int cpumf_pmu_add(struct perf_event *event, int flags)
542435 {
543
-	struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
436
+	struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
544437 
545438 	/* Check authorization for the counter set to which this
546439 	 * counter belongs.
....@@ -564,7 +457,7 @@
564457 
565458 static void cpumf_pmu_del(struct perf_event *event, int flags)
566459 {
567
-	struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
460
+	struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
568461 
569462 	cpumf_pmu_stop(event, PERF_EF_UPDATE);
570463 
....@@ -592,7 +485,7 @@
592485  */
593486 static void cpumf_pmu_start_txn(struct pmu *pmu, unsigned int txn_flags)
594487 {
595
-	struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
488
+	struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
596489 
597490 	WARN_ON_ONCE(cpuhw->txn_flags);		/* txn already in flight */
598491 
....@@ -612,7 +505,7 @@
612505 static void cpumf_pmu_cancel_txn(struct pmu *pmu)
613506 {
614507 	unsigned int txn_flags;
615
-	struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
508
+	struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
616509 
617510 	WARN_ON_ONCE(!cpuhw->txn_flags);	/* no txn in flight */
618511 
....@@ -633,7 +526,7 @@
633526  */
634527 static int cpumf_pmu_commit_txn(struct pmu *pmu)
635528 {
636
-	struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events);
529
+	struct cpu_cf_events *cpuhw = this_cpu_ptr(&cpu_cf_events);
637530 	u64 state;
638531 
639532 	WARN_ON_ONCE(!cpuhw->txn_flags);	/* no txn in flight */
....@@ -671,54 +564,17 @@
671564 	.cancel_txn   = cpumf_pmu_cancel_txn,
672565 };
673566 
674
-static int cpumf_pmf_setup(unsigned int cpu, int flags)
675
-{
676
-	local_irq_disable();
677
-	setup_pmc_cpu(&flags);
678
-	local_irq_enable();
679
-	return 0;
680
-}
681
-
682
-static int s390_pmu_online_cpu(unsigned int cpu)
683
-{
684
-	return cpumf_pmf_setup(cpu, PMC_INIT);
685
-}
686
-
687
-static int s390_pmu_offline_cpu(unsigned int cpu)
688
-{
689
-	return cpumf_pmf_setup(cpu, PMC_RELEASE);
690
-}
691
-
692567 static int __init cpumf_pmu_init(void)
693568 {
694569 	int rc;
695570 
696
-	if (!cpum_cf_avail())
571
+	if (!kernel_cpumcf_avail())
697572 		return -ENODEV;
698573 
699
-	/* clear bit 15 of cr0 to unauthorize problem-state to
700
-	 * extract measurement counters */
701
-	ctl_clear_bit(0, 48);
702
-
703
-	/* register handler for measurement-alert interruptions */
704
-	rc = register_external_irq(EXT_IRQ_MEASURE_ALERT,
705
-				   cpumf_measurement_alert);
706
-	if (rc) {
707
-		pr_err("Registering for CPU-measurement alerts "
708
-		       "failed with rc=%i\n", rc);
709
-		return rc;
710
-	}
711
-
712574 	cpumf_pmu.attr_groups = cpumf_cf_event_group();
713
-	rc = perf_pmu_register(&cpumf_pmu, "cpum_cf", PERF_TYPE_RAW);
714
-	if (rc) {
575
+	rc = perf_pmu_register(&cpumf_pmu, "cpum_cf", -1);
576
+	if (rc)
715577 		pr_err("Registering the cpum_cf PMU failed with rc=%i\n", rc);
716
-		unregister_external_irq(EXT_IRQ_MEASURE_ALERT,
717
-					cpumf_measurement_alert);
718
-		return rc;
719
-	}
720
-	return cpuhp_setup_state(CPUHP_AP_PERF_S390_CF_ONLINE,
721
-				 "perf/s390/cf:online",
722
-				 s390_pmu_online_cpu, s390_pmu_offline_cpu);
578
+	return rc;
723579 }
724
-early_initcall(cpumf_pmu_init);
580
+subsys_initcall(cpumf_pmu_init);