~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,14 +1,10 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
1	2	/*
2	3	* In-Memory Collection (IMC) Performance Monitor counter support.
3	4	*
4	5	* Copyright (C) 2017 Madhavan Srinivasan, IBM Corporation.
5	6	* (C) 2017 Anju T Sudhakar, IBM Corporation.
6	7	* (C) 2017 Hemant K Shaw, IBM Corporation.
7		- *
8		- * This program is free software; you can redistribute it and/or
9		- * modify it under the terms of the GNU General Public License
10		- * as published by the Free Software Foundation; either version
11		- * 2 of the License, or later version.
12	8	*/
13	9	#include <linux/perf_event.h>
14	10	#include <linux/slab.h>
..	..	@@ -17,6 +13,7 @@
17	13	#include <asm/cputhreads.h>
18	14	#include <asm/smp.h>
19	15	#include <linux/string.h>
	16	+#include <linux/spinlock.h>
20	17
21	18	/* Nest IMC data structures and variables */
22	19
..	..	@@ -28,13 +25,13 @@
28	25	static DEFINE_PER_CPU(struct imc_pmu_ref *, local_nest_imc_refc);
29	26	static struct imc_pmu **per_nest_pmu_arr;
30	27	static cpumask_t nest_imc_cpumask;
31		-struct imc_pmu_ref *nest_imc_refc;
	28	+static struct imc_pmu_ref *nest_imc_refc;
32	29	static int nest_pmus;
33	30
34	31	/* Core IMC data structures and variables */
35	32
36	33	static cpumask_t core_imc_cpumask;
37		-struct imc_pmu_ref *core_imc_refc;
	34	+static struct imc_pmu_ref *core_imc_refc;
38	35	static struct imc_pmu *core_imc_pmu;
39	36
40	37	/* Thread IMC data structures and variables */
..	..	@@ -43,12 +40,27 @@
43	40	static struct imc_pmu *thread_imc_pmu;
44	41	static int thread_imc_mem_size;
45	42
46		-struct imc_pmu imc_event_to_pmu(struct perf_event event)
	43	+/* Trace IMC data structures */
	44	+static DEFINE_PER_CPU(u64 *, trace_imc_mem);
	45	+static struct imc_pmu_ref *trace_imc_refc;
	46	+static int trace_imc_mem_size;
	47	+
	48	+/*
	49	+ * Global data structure used to avoid races between thread,
	50	+ * core and trace-imc
	51	+ */
	52	+static struct imc_pmu_ref imc_global_refc = {
	53	+ .lock = __SPIN_LOCK_INITIALIZER(imc_global_refc.lock),
	54	+ .id = 0,
	55	+ .refc = 0,
	56	+};
	57	+
	58	+static struct imc_pmu imc_event_to_pmu(struct perf_event event)
47	59	{
48	60	return container_of(event->pmu, struct imc_pmu, pmu);
49	61	}
50	62
51		-PMU_FORMAT_ATTR(event, "config:0-40");
	63	+PMU_FORMAT_ATTR(event, "config:0-61");
52	64	PMU_FORMAT_ATTR(offset, "config:0-31");
53	65	PMU_FORMAT_ATTR(rvalue, "config:32");
54	66	PMU_FORMAT_ATTR(mode, "config:33-40");
..	..	@@ -63,6 +75,25 @@
63	75	static struct attribute_group imc_format_group = {
64	76	.name = "format",
65	77	.attrs = imc_format_attrs,
	78	+};
	79	+
	80	+/* Format attribute for imc trace-mode */
	81	+PMU_FORMAT_ATTR(cpmc_reserved, "config:0-19");
	82	+PMU_FORMAT_ATTR(cpmc_event, "config:20-27");
	83	+PMU_FORMAT_ATTR(cpmc_samplesel, "config:28-29");
	84	+PMU_FORMAT_ATTR(cpmc_load, "config:30-61");
	85	+static struct attribute *trace_imc_format_attrs[] = {
	86	+ &format_attr_event.attr,
	87	+ &format_attr_cpmc_reserved.attr,
	88	+ &format_attr_cpmc_event.attr,
	89	+ &format_attr_cpmc_samplesel.attr,
	90	+ &format_attr_cpmc_load.attr,
	91	+ NULL,
	92	+};
	93	+
	94	+static struct attribute_group trace_imc_format_group = {
	95	+.name = "format",
	96	+.attrs = trace_imc_format_attrs,
66	97	};
67	98
68	99	/* Get the cpumask printed to a buffer "buf" */
..	..	@@ -342,7 +373,14 @@
342	373	*/
343	374	nid = cpu_to_node(cpu);
344	375	l_cpumask = cpumask_of_node(nid);
345		- target = cpumask_any_but(l_cpumask, cpu);
	376	+ target = cpumask_last(l_cpumask);
	377	+
	378	+ /*
	379	+ * If this(target) is the last cpu in the cpumask for this chip,
	380	+ * check for any possible online cpu in the chip.
	381	+ */
	382	+ if (unlikely(target == cpu))
	383	+ target = cpumask_any_but(l_cpumask, cpu);
346	384
347	385	/*
348	386	* Update the cpumask with the target cpu and
..	..	@@ -356,7 +394,7 @@
356	394	get_hard_smp_processor_id(cpu));
357	395	/*
358	396	* If this is the last cpu in this chip then, skip the reference
359		- * count mutex lock and make the reference count on this chip zero.
	397	+ * count lock and make the reference count on this chip zero.
360	398	*/
361	399	ref = get_nest_pmu_ref(cpu);
362	400	if (!ref)
..	..	@@ -418,15 +456,15 @@
418	456	/*
419	457	* See if we need to disable the nest PMU.
420	458	* If no events are currently in use, then we have to take a
421		- * mutex to ensure that we don't race with another task doing
	459	+ * lock to ensure that we don't race with another task doing
422	460	* enable or disable the nest counters.
423	461	*/
424	462	ref = get_nest_pmu_ref(event->cpu);
425	463	if (!ref)
426	464	return;
427	465
428		- /* Take the mutex lock for this node and then decrement the reference count */
429		- mutex_lock(&ref->lock);
	466	+ /* Take the lock for this node and then decrement the reference count */
	467	+ spin_lock(&ref->lock);
430	468	if (ref->refc == 0) {
431	469	/*
432	470	* The scenario where this is true is, when perf session is
..	..	@@ -438,7 +476,7 @@
438	476	* an OPAL call to disable the engine in that node.
439	477	*
440	478	*/
441		- mutex_unlock(&ref->lock);
	479	+ spin_unlock(&ref->lock);
442	480	return;
443	481	}
444	482	ref->refc--;
..	..	@@ -446,7 +484,7 @@
446	484	rc = opal_imc_counters_stop(OPAL_IMC_COUNTERS_NEST,
447	485	get_hard_smp_processor_id(event->cpu));
448	486	if (rc) {
449		- mutex_unlock(&ref->lock);
	487	+ spin_unlock(&ref->lock);
450	488	pr_err("nest-imc: Unable to stop the counters for core %d\n", node_id);
451	489	return;
452	490	}
..	..	@@ -454,7 +492,7 @@
454	492	WARN(1, "nest-imc: Invalid event reference count\n");
455	493	ref->refc = 0;
456	494	}
457		- mutex_unlock(&ref->lock);
	495	+ spin_unlock(&ref->lock);
458	496	}
459	497
460	498	static int nest_imc_event_init(struct perf_event *event)
..	..	@@ -471,15 +509,6 @@
471	509
472	510	/* Sampling not supported */
473	511	if (event->hw.sample_period)
474		- return -EINVAL;
475		-
476		- /* unsupported modes and filters */
477		- if (event->attr.exclude_user \|\|
478		- event->attr.exclude_kernel \|\|
479		- event->attr.exclude_hv \|\|
480		- event->attr.exclude_idle \|\|
481		- event->attr.exclude_host \|\|
482		- event->attr.exclude_guest)
483	512	return -EINVAL;
484	513
485	514	if (event->cpu < 0)
..	..	@@ -522,26 +551,25 @@
522	551
523	552	/*
524	553	* Get the imc_pmu_ref struct for this node.
525		- * Take the mutex lock and then increment the count of nest pmu events
526		- * inited.
	554	+ * Take the lock and then increment the count of nest pmu events inited.
527	555	*/
528	556	ref = get_nest_pmu_ref(event->cpu);
529	557	if (!ref)
530	558	return -EINVAL;
531	559
532		- mutex_lock(&ref->lock);
	560	+ spin_lock(&ref->lock);
533	561	if (ref->refc == 0) {
534	562	rc = opal_imc_counters_start(OPAL_IMC_COUNTERS_NEST,
535	563	get_hard_smp_processor_id(event->cpu));
536	564	if (rc) {
537		- mutex_unlock(&ref->lock);
	565	+ spin_unlock(&ref->lock);
538	566	pr_err("nest-imc: Unable to start the counters for node %d\n",
539	567	node_id);
540	568	return rc;
541	569	}
542	570	}
543	571	++ref->refc;
544		- mutex_unlock(&ref->lock);
	572	+ spin_unlock(&ref->lock);
545	573
546	574	event->destroy = nest_imc_counters_release;
547	575	return 0;
..	..	@@ -559,6 +587,7 @@
559	587	{
560	588	int nid, rc = 0, core_id = (cpu / threads_per_core);
561	589	struct imc_mem_info *mem_info;
	590	+ struct page *page;
562	591
563	592	/*
564	593	* alloc_pages_node() will allocate memory for core in the
..	..	@@ -569,15 +598,15 @@
569	598	mem_info->id = core_id;
570	599
571	600	/* We need only vbase for core counters */
572		- mem_info->vbase = page_address(alloc_pages_node(nid,
573		- GFP_KERNEL \| __GFP_ZERO \| __GFP_THISNODE \|
574		- __GFP_NOWARN, get_order(size)));
575		- if (!mem_info->vbase)
	601	+ page = alloc_pages_node(nid,
	602	+ GFP_KERNEL \| __GFP_ZERO \| __GFP_THISNODE \|
	603	+ __GFP_NOWARN, get_order(size));
	604	+ if (!page)
576	605	return -ENOMEM;
	606	+ mem_info->vbase = page_address(page);
577	607
578		- /* Init the mutex */
579	608	core_imc_refc[core_id].id = core_id;
580		- mutex_init(&core_imc_refc[core_id].lock);
	609	+ spin_lock_init(&core_imc_refc[core_id].lock);
581	610
582	611	rc = opal_imc_counters_init(OPAL_IMC_COUNTERS_CORE,
583	612	__pa((void *)mem_info->vbase),
..	..	@@ -656,16 +685,18 @@
656	685	return 0;
657	686
658	687	/* Find any online cpu in that core except the current "cpu" */
659		- ncpu = cpumask_any_but(cpu_sibling_mask(cpu), cpu);
	688	+ ncpu = cpumask_last(cpu_sibling_mask(cpu));
	689	+
	690	+ if (unlikely(ncpu == cpu))
	691	+ ncpu = cpumask_any_but(cpu_sibling_mask(cpu), cpu);
660	692
661	693	if (ncpu >= 0 && ncpu < nr_cpu_ids) {
662	694	cpumask_set_cpu(ncpu, &core_imc_cpumask);
663	695	perf_pmu_migrate_context(&core_imc_pmu->pmu, cpu, ncpu);
664	696	} else {
665	697	/*
666		- * If this is the last cpu in this core then, skip taking refernce
667		- * count mutex lock for this core and directly zero "refc" for
668		- * this core.
	698	+ * If this is the last cpu in this core then skip taking reference
	699	+ * count lock for this core and directly zero "refc" for this core.
669	700	*/
670	701	opal_imc_counters_stop(OPAL_IMC_COUNTERS_CORE,
671	702	get_hard_smp_processor_id(cpu));
..	..	@@ -675,6 +706,16 @@
675	706	return -EINVAL;
676	707
677	708	ref->refc = 0;
	709	+ /*
	710	+ * Reduce the global reference count, if this is the
	711	+ * last cpu in this core and core-imc event running
	712	+ * in this cpu.
	713	+ */
	714	+ spin_lock(&imc_global_refc.lock);
	715	+ if (imc_global_refc.id == IMC_DOMAIN_CORE)
	716	+ imc_global_refc.refc--;
	717	+
	718	+ spin_unlock(&imc_global_refc.lock);
678	719	}
679	720	return 0;
680	721	}
..	..	@@ -687,6 +728,23 @@
687	728	ppc_core_imc_cpu_offline);
688	729	}
689	730
	731	+static void reset_global_refc(struct perf_event *event)
	732	+{
	733	+ spin_lock(&imc_global_refc.lock);
	734	+ imc_global_refc.refc--;
	735	+
	736	+ /*
	737	+ * If no other thread is running any
	738	+ * event for this domain(thread/core/trace),
	739	+ * set the global id to zero.
	740	+ */
	741	+ if (imc_global_refc.refc <= 0) {
	742	+ imc_global_refc.refc = 0;
	743	+ imc_global_refc.id = 0;
	744	+ }
	745	+ spin_unlock(&imc_global_refc.lock);
	746	+}
	747	+
690	748	static void core_imc_counters_release(struct perf_event *event)
691	749	{
692	750	int rc, core_id;
..	..	@@ -697,17 +755,17 @@
697	755	/*
698	756	* See if we need to disable the IMC PMU.
699	757	* If no events are currently in use, then we have to take a
700		- * mutex to ensure that we don't race with another task doing
	758	+ * lock to ensure that we don't race with another task doing
701	759	* enable or disable the core counters.
702	760	*/
703	761	core_id = event->cpu / threads_per_core;
704	762
705		- /* Take the mutex lock and decrement the refernce count for this core */
	763	+ /* Take the lock and decrement the refernce count for this core */
706	764	ref = &core_imc_refc[core_id];
707	765	if (!ref)
708	766	return;
709	767
710		- mutex_lock(&ref->lock);
	768	+ spin_lock(&ref->lock);
711	769	if (ref->refc == 0) {
712	770	/*
713	771	* The scenario where this is true is, when perf session is
..	..	@@ -719,7 +777,7 @@
719	777	* an OPAL call to disable the engine in that core.
720	778	*
721	779	*/
722		- mutex_unlock(&ref->lock);
	780	+ spin_unlock(&ref->lock);
723	781	return;
724	782	}
725	783	ref->refc--;
..	..	@@ -727,7 +785,7 @@
727	785	rc = opal_imc_counters_stop(OPAL_IMC_COUNTERS_CORE,
728	786	get_hard_smp_processor_id(event->cpu));
729	787	if (rc) {
730		- mutex_unlock(&ref->lock);
	788	+ spin_unlock(&ref->lock);
731	789	pr_err("IMC: Unable to stop the counters for core %d\n", core_id);
732	790	return;
733	791	}
..	..	@@ -735,7 +793,9 @@
735	793	WARN(1, "core-imc: Invalid event reference count\n");
736	794	ref->refc = 0;
737	795	}
738		- mutex_unlock(&ref->lock);
	796	+ spin_unlock(&ref->lock);
	797	+
	798	+ reset_global_refc(event);
739	799	}
740	800
741	801	static int core_imc_event_init(struct perf_event *event)
..	..	@@ -751,15 +811,6 @@
751	811
752	812	/* Sampling not supported */
753	813	if (event->hw.sample_period)
754		- return -EINVAL;
755		-
756		- /* unsupported modes and filters */
757		- if (event->attr.exclude_user \|\|
758		- event->attr.exclude_kernel \|\|
759		- event->attr.exclude_hv \|\|
760		- event->attr.exclude_idle \|\|
761		- event->attr.exclude_host \|\|
762		- event->attr.exclude_guest)
763	814	return -EINVAL;
764	815
765	816	if (event->cpu < 0)
..	..	@@ -780,7 +831,6 @@
780	831	if ((!pcmi->vbase))
781	832	return -ENODEV;
782	833
783		- /* Get the core_imc mutex for this core */
784	834	ref = &core_imc_refc[core_id];
785	835	if (!ref)
786	836	return -EINVAL;
..	..	@@ -788,22 +838,45 @@
788	838	/*
789	839	* Core pmu units are enabled only when it is used.
790	840	* See if this is triggered for the first time.
791		- * If yes, take the mutex lock and enable the core counters.
	841	+ * If yes, take the lock and enable the core counters.
792	842	* If not, just increment the count in core_imc_refc struct.
793	843	*/
794		- mutex_lock(&ref->lock);
	844	+ spin_lock(&ref->lock);
795	845	if (ref->refc == 0) {
796	846	rc = opal_imc_counters_start(OPAL_IMC_COUNTERS_CORE,
797	847	get_hard_smp_processor_id(event->cpu));
798	848	if (rc) {
799		- mutex_unlock(&ref->lock);
	849	+ spin_unlock(&ref->lock);
800	850	pr_err("core-imc: Unable to start the counters for core %d\n",
801	851	core_id);
802	852	return rc;
803	853	}
804	854	}
805	855	++ref->refc;
806		- mutex_unlock(&ref->lock);
	856	+ spin_unlock(&ref->lock);
	857	+
	858	+ /*
	859	+ * Since the system can run either in accumulation or trace-mode
	860	+ * of IMC at a time, core-imc events are allowed only if no other
	861	+ * trace/thread imc events are enabled/monitored.
	862	+ *
	863	+ * Take the global lock, and check the refc.id
	864	+ * to know whether any other trace/thread imc
	865	+ * events are running.
	866	+ */
	867	+ spin_lock(&imc_global_refc.lock);
	868	+ if (imc_global_refc.id == 0 \|\| imc_global_refc.id == IMC_DOMAIN_CORE) {
	869	+ /*
	870	+ * No other trace/thread imc events are running in
	871	+ * the system, so set the refc.id to core-imc.
	872	+ */
	873	+ imc_global_refc.id = IMC_DOMAIN_CORE;
	874	+ imc_global_refc.refc++;
	875	+ } else {
	876	+ spin_unlock(&imc_global_refc.lock);
	877	+ return -EBUSY;
	878	+ }
	879	+ spin_unlock(&imc_global_refc.lock);
807	880
808	881	event->hw.event_base = (u64)pcmi->vbase + (config & IMC_EVENT_OFFSET_MASK);
809	882	event->destroy = core_imc_counters_release;
..	..	@@ -811,8 +884,11 @@
811	884	}
812	885
813	886	/*
814		- * Allocates a page of memory for each of the online cpus, and write the
815		- * physical base address of that page to the LDBAR for that cpu.
	887	+ * Allocates a page of memory for each of the online cpus, and load
	888	+ * LDBAR with 0.
	889	+ * The physical base address of the page allocated for a cpu will be
	890	+ * written to the LDBAR for that cpu, when the thread-imc event
	891	+ * is added.
816	892	*
817	893	* LDBAR Register Layout:
818	894	*
..	..	@@ -830,26 +906,26 @@
830	906	*/
831	907	static int thread_imc_mem_alloc(int cpu_id, int size)
832	908	{
833		- u64 ldbar_value, *local_mem = per_cpu(thread_imc_mem, cpu_id);
	909	+ u64 *local_mem = per_cpu(thread_imc_mem, cpu_id);
834	910	int nid = cpu_to_node(cpu_id);
835	911
836	912	if (!local_mem) {
	913	+ struct page *page;
837	914	/*
838	915	* This case could happen only once at start, since we dont
839	916	* free the memory in cpu offline path.
840	917	*/
841		- local_mem = page_address(alloc_pages_node(nid,
	918	+ page = alloc_pages_node(nid,
842	919	GFP_KERNEL \| __GFP_ZERO \| __GFP_THISNODE \|
843		- __GFP_NOWARN, get_order(size)));
844		- if (!local_mem)
	920	+ __GFP_NOWARN, get_order(size));
	921	+ if (!page)
845	922	return -ENOMEM;
	923	+ local_mem = page_address(page);
846	924
847	925	per_cpu(thread_imc_mem, cpu_id) = local_mem;
848	926	}
849	927
850		- ldbar_value = ((u64)local_mem & THREAD_IMC_LDBAR_MASK) \| THREAD_IMC_ENABLE;
851		-
852		- mtspr(SPRN_LDBAR, ldbar_value);
	928	+ mtspr(SPRN_LDBAR, 0);
853	929	return 0;
854	930	}
855	931
..	..	@@ -860,7 +936,23 @@
860	936
861	937	static int ppc_thread_imc_cpu_offline(unsigned int cpu)
862	938	{
863		- mtspr(SPRN_LDBAR, 0);
	939	+ /*
	940	+ * Set the bit 0 of LDBAR to zero.
	941	+ *
	942	+ * If bit 0 of LDBAR is unset, it will stop posting
	943	+ * the counter data to memory.
	944	+ * For thread-imc, bit 0 of LDBAR will be set to 1 in the
	945	+ * event_add function. So reset this bit here, to stop the updates
	946	+ * to memory in the cpu_offline path.
	947	+ */
	948	+ mtspr(SPRN_LDBAR, (mfspr(SPRN_LDBAR) & (~(1UL << 63))));
	949	+
	950	+ /* Reduce the refc if thread-imc event running on this cpu */
	951	+ spin_lock(&imc_global_refc.lock);
	952	+ if (imc_global_refc.id == IMC_DOMAIN_THREAD)
	953	+ imc_global_refc.refc--;
	954	+ spin_unlock(&imc_global_refc.lock);
	955	+
864	956	return 0;
865	957	}
866	958
..	..	@@ -881,6 +973,9 @@
881	973	if (event->attr.type != event->pmu->type)
882	974	return -ENOENT;
883	975
	976	+ if (!perfmon_capable())
	977	+ return -EACCES;
	978	+
884	979	/* Sampling not supported */
885	980	if (event->hw.sample_period)
886	981	return -EINVAL;
..	..	@@ -896,7 +991,22 @@
896	991	if (!target)
897	992	return -EINVAL;
898	993
	994	+ spin_lock(&imc_global_refc.lock);
	995	+ /*
	996	+ * Check if any other trace/core imc events are running in the
	997	+ * system, if not set the global id to thread-imc.
	998	+ */
	999	+ if (imc_global_refc.id == 0 \|\| imc_global_refc.id == IMC_DOMAIN_THREAD) {
	1000	+ imc_global_refc.id = IMC_DOMAIN_THREAD;
	1001	+ imc_global_refc.refc++;
	1002	+ } else {
	1003	+ spin_unlock(&imc_global_refc.lock);
	1004	+ return -EBUSY;
	1005	+ }
	1006	+ spin_unlock(&imc_global_refc.lock);
	1007	+
899	1008	event->pmu->task_ctx_nr = perf_sw_context;
	1009	+ event->destroy = reset_global_refc;
900	1010	return 0;
901	1011	}
902	1012
..	..	@@ -1000,6 +1110,7 @@
1000	1110	{
1001	1111	int core_id;
1002	1112	struct imc_pmu_ref *ref;
	1113	+ u64 ldbar_value, *local_mem = per_cpu(thread_imc_mem, smp_processor_id());
1003	1114
1004	1115	if (flags & PERF_EF_START)
1005	1116	imc_event_start(event, flags);
..	..	@@ -1008,28 +1119,31 @@
1008	1119	return -EINVAL;
1009	1120
1010	1121	core_id = smp_processor_id() / threads_per_core;
	1122	+ ldbar_value = ((u64)local_mem & THREAD_IMC_LDBAR_MASK) \| THREAD_IMC_ENABLE;
	1123	+ mtspr(SPRN_LDBAR, ldbar_value);
	1124	+
1011	1125	/*
1012	1126	* imc pmus are enabled only when it is used.
1013	1127	* See if this is triggered for the first time.
1014		- * If yes, take the mutex lock and enable the counters.
	1128	+ * If yes, take the lock and enable the counters.
1015	1129	* If not, just increment the count in ref count struct.
1016	1130	*/
1017	1131	ref = &core_imc_refc[core_id];
1018	1132	if (!ref)
1019	1133	return -EINVAL;
1020	1134
1021		- mutex_lock(&ref->lock);
	1135	+ spin_lock(&ref->lock);
1022	1136	if (ref->refc == 0) {
1023	1137	if (opal_imc_counters_start(OPAL_IMC_COUNTERS_CORE,
1024	1138	get_hard_smp_processor_id(smp_processor_id()))) {
1025		- mutex_unlock(&ref->lock);
	1139	+ spin_unlock(&ref->lock);
1026	1140	pr_err("thread-imc: Unable to start the counter\
1027	1141	for core %d\n", core_id);
1028	1142	return -EINVAL;
1029	1143	}
1030	1144	}
1031	1145	++ref->refc;
1032		- mutex_unlock(&ref->lock);
	1146	+ spin_unlock(&ref->lock);
1033	1147	return 0;
1034	1148	}
1035	1149
..	..	@@ -1039,21 +1153,19 @@
1039	1153	int core_id;
1040	1154	struct imc_pmu_ref *ref;
1041	1155
1042		- /*
1043		- * Take a snapshot and calculate the delta and update
1044		- * the event counter values.
1045		- */
1046		- imc_event_update(event);
1047		-
1048	1156	core_id = smp_processor_id() / threads_per_core;
1049	1157	ref = &core_imc_refc[core_id];
	1158	+ if (!ref) {
	1159	+ pr_debug("imc: Failed to get event reference count\n");
	1160	+ return;
	1161	+ }
1050	1162
1051		- mutex_lock(&ref->lock);
	1163	+ spin_lock(&ref->lock);
1052	1164	ref->refc--;
1053	1165	if (ref->refc == 0) {
1054	1166	if (opal_imc_counters_stop(OPAL_IMC_COUNTERS_CORE,
1055	1167	get_hard_smp_processor_id(smp_processor_id()))) {
1056		- mutex_unlock(&ref->lock);
	1168	+ spin_unlock(&ref->lock);
1057	1169	pr_err("thread-imc: Unable to stop the counters\
1058	1170	for core %d\n", core_id);
1059	1171	return;
..	..	@@ -1061,7 +1173,293 @@
1061	1173	} else if (ref->refc < 0) {
1062	1174	ref->refc = 0;
1063	1175	}
1064		- mutex_unlock(&ref->lock);
	1176	+ spin_unlock(&ref->lock);
	1177	+
	1178	+ /* Set bit 0 of LDBAR to zero, to stop posting updates to memory */
	1179	+ mtspr(SPRN_LDBAR, (mfspr(SPRN_LDBAR) & (~(1UL << 63))));
	1180	+
	1181	+ /*
	1182	+ * Take a snapshot and calculate the delta and update
	1183	+ * the event counter values.
	1184	+ */
	1185	+ imc_event_update(event);
	1186	+}
	1187	+
	1188	+/*
	1189	+ * Allocate a page of memory for each cpu, and load LDBAR with 0.
	1190	+ */
	1191	+static int trace_imc_mem_alloc(int cpu_id, int size)
	1192	+{
	1193	+ u64 *local_mem = per_cpu(trace_imc_mem, cpu_id);
	1194	+ int phys_id = cpu_to_node(cpu_id), rc = 0;
	1195	+ int core_id = (cpu_id / threads_per_core);
	1196	+
	1197	+ if (!local_mem) {
	1198	+ struct page *page;
	1199	+
	1200	+ page = alloc_pages_node(phys_id,
	1201	+ GFP_KERNEL \| __GFP_ZERO \| __GFP_THISNODE \|
	1202	+ __GFP_NOWARN, get_order(size));
	1203	+ if (!page)
	1204	+ return -ENOMEM;
	1205	+ local_mem = page_address(page);
	1206	+ per_cpu(trace_imc_mem, cpu_id) = local_mem;
	1207	+
	1208	+ /* Initialise the counters for trace mode */
	1209	+ rc = opal_imc_counters_init(OPAL_IMC_COUNTERS_TRACE, __pa((void *)local_mem),
	1210	+ get_hard_smp_processor_id(cpu_id));
	1211	+ if (rc) {
	1212	+ pr_info("IMC:opal init failed for trace imc\n");
	1213	+ return rc;
	1214	+ }
	1215	+ }
	1216	+
	1217	+ trace_imc_refc[core_id].id = core_id;
	1218	+ spin_lock_init(&trace_imc_refc[core_id].lock);
	1219	+
	1220	+ mtspr(SPRN_LDBAR, 0);
	1221	+ return 0;
	1222	+}
	1223	+
	1224	+static int ppc_trace_imc_cpu_online(unsigned int cpu)
	1225	+{
	1226	+ return trace_imc_mem_alloc(cpu, trace_imc_mem_size);
	1227	+}
	1228	+
	1229	+static int ppc_trace_imc_cpu_offline(unsigned int cpu)
	1230	+{
	1231	+ /*
	1232	+ * No need to set bit 0 of LDBAR to zero, as
	1233	+ * it is set to zero for imc trace-mode
	1234	+ *
	1235	+ * Reduce the refc if any trace-imc event running
	1236	+ * on this cpu.
	1237	+ */
	1238	+ spin_lock(&imc_global_refc.lock);
	1239	+ if (imc_global_refc.id == IMC_DOMAIN_TRACE)
	1240	+ imc_global_refc.refc--;
	1241	+ spin_unlock(&imc_global_refc.lock);
	1242	+
	1243	+ return 0;
	1244	+}
	1245	+
	1246	+static int trace_imc_cpu_init(void)
	1247	+{
	1248	+ return cpuhp_setup_state(CPUHP_AP_PERF_POWERPC_TRACE_IMC_ONLINE,
	1249	+ "perf/powerpc/imc_trace:online",
	1250	+ ppc_trace_imc_cpu_online,
	1251	+ ppc_trace_imc_cpu_offline);
	1252	+}
	1253	+
	1254	+static u64 get_trace_imc_event_base_addr(void)
	1255	+{
	1256	+ return (u64)per_cpu(trace_imc_mem, smp_processor_id());
	1257	+}
	1258	+
	1259	+/*
	1260	+ * Function to parse trace-imc data obtained
	1261	+ * and to prepare the perf sample.
	1262	+ */
	1263	+static int trace_imc_prepare_sample(struct trace_imc_data *mem,
	1264	+ struct perf_sample_data *data,
	1265	+ u64 *prev_tb,
	1266	+ struct perf_event_header *header,
	1267	+ struct perf_event *event)
	1268	+{
	1269	+ /* Sanity checks for a valid record */
	1270	+ if (be64_to_cpu(READ_ONCE(mem->tb1)) > *prev_tb)
	1271	+ *prev_tb = be64_to_cpu(READ_ONCE(mem->tb1));
	1272	+ else
	1273	+ return -EINVAL;
	1274	+
	1275	+ if ((be64_to_cpu(READ_ONCE(mem->tb1)) & IMC_TRACE_RECORD_TB1_MASK) !=
	1276	+ be64_to_cpu(READ_ONCE(mem->tb2)))
	1277	+ return -EINVAL;
	1278	+
	1279	+ /* Prepare perf sample */
	1280	+ data->ip = be64_to_cpu(READ_ONCE(mem->ip));
	1281	+ data->period = event->hw.last_period;
	1282	+
	1283	+ header->type = PERF_RECORD_SAMPLE;
	1284	+ header->size = sizeof(*header) + event->header_size;
	1285	+ header->misc = 0;
	1286	+
	1287	+ if (cpu_has_feature(CPU_FTR_ARCH_31)) {
	1288	+ switch (IMC_TRACE_RECORD_VAL_HVPR(be64_to_cpu(READ_ONCE(mem->val)))) {
	1289	+ case 0:/* when MSR HV and PR not set in the trace-record */
	1290	+ header->misc \|= PERF_RECORD_MISC_GUEST_KERNEL;
	1291	+ break;
	1292	+ case 1: /* MSR HV is 0 and PR is 1 */
	1293	+ header->misc \|= PERF_RECORD_MISC_GUEST_USER;
	1294	+ break;
	1295	+ case 2: /* MSR HV is 1 and PR is 0 */
	1296	+ header->misc \|= PERF_RECORD_MISC_KERNEL;
	1297	+ break;
	1298	+ case 3: /* MSR HV is 1 and PR is 1 */
	1299	+ header->misc \|= PERF_RECORD_MISC_USER;
	1300	+ break;
	1301	+ default:
	1302	+ pr_info("IMC: Unable to set the flag based on MSR bits\n");
	1303	+ break;
	1304	+ }
	1305	+ } else {
	1306	+ if (is_kernel_addr(data->ip))
	1307	+ header->misc \|= PERF_RECORD_MISC_KERNEL;
	1308	+ else
	1309	+ header->misc \|= PERF_RECORD_MISC_USER;
	1310	+ }
	1311	+ perf_event_header__init_id(header, data, event);
	1312	+
	1313	+ return 0;
	1314	+}
	1315	+
	1316	+static void dump_trace_imc_data(struct perf_event *event)
	1317	+{
	1318	+ struct trace_imc_data *mem;
	1319	+ int i, ret;
	1320	+ u64 prev_tb = 0;
	1321	+
	1322	+ mem = (struct trace_imc_data *)get_trace_imc_event_base_addr();
	1323	+ for (i = 0; i < (trace_imc_mem_size / sizeof(struct trace_imc_data));
	1324	+ i++, mem++) {
	1325	+ struct perf_sample_data data;
	1326	+ struct perf_event_header header;
	1327	+
	1328	+ ret = trace_imc_prepare_sample(mem, &data, &prev_tb, &header, event);
	1329	+ if (ret) /* Exit, if not a valid record */
	1330	+ break;
	1331	+ else {
	1332	+ /* If this is a valid record, create the sample */
	1333	+ struct perf_output_handle handle;
	1334	+
	1335	+ if (perf_output_begin(&handle, &data, event, header.size))
	1336	+ return;
	1337	+
	1338	+ perf_output_sample(&handle, &header, &data, event);
	1339	+ perf_output_end(&handle);
	1340	+ }
	1341	+ }
	1342	+}
	1343	+
	1344	+static int trace_imc_event_add(struct perf_event *event, int flags)
	1345	+{
	1346	+ int core_id = smp_processor_id() / threads_per_core;
	1347	+ struct imc_pmu_ref *ref = NULL;
	1348	+ u64 local_mem, ldbar_value;
	1349	+
	1350	+ /* Set trace-imc bit in ldbar and load ldbar with per-thread memory address */
	1351	+ local_mem = get_trace_imc_event_base_addr();
	1352	+ ldbar_value = ((u64)local_mem & THREAD_IMC_LDBAR_MASK) \| TRACE_IMC_ENABLE;
	1353	+
	1354	+ /* trace-imc reference count */
	1355	+ if (trace_imc_refc)
	1356	+ ref = &trace_imc_refc[core_id];
	1357	+ if (!ref) {
	1358	+ pr_debug("imc: Failed to get the event reference count\n");
	1359	+ return -EINVAL;
	1360	+ }
	1361	+
	1362	+ mtspr(SPRN_LDBAR, ldbar_value);
	1363	+ spin_lock(&ref->lock);
	1364	+ if (ref->refc == 0) {
	1365	+ if (opal_imc_counters_start(OPAL_IMC_COUNTERS_TRACE,
	1366	+ get_hard_smp_processor_id(smp_processor_id()))) {
	1367	+ spin_unlock(&ref->lock);
	1368	+ pr_err("trace-imc: Unable to start the counters for core %d\n", core_id);
	1369	+ return -EINVAL;
	1370	+ }
	1371	+ }
	1372	+ ++ref->refc;
	1373	+ spin_unlock(&ref->lock);
	1374	+ return 0;
	1375	+}
	1376	+
	1377	+static void trace_imc_event_read(struct perf_event *event)
	1378	+{
	1379	+ return;
	1380	+}
	1381	+
	1382	+static void trace_imc_event_stop(struct perf_event *event, int flags)
	1383	+{
	1384	+ u64 local_mem = get_trace_imc_event_base_addr();
	1385	+ dump_trace_imc_data(event);
	1386	+ memset((void *)local_mem, 0, sizeof(u64));
	1387	+}
	1388	+
	1389	+static void trace_imc_event_start(struct perf_event *event, int flags)
	1390	+{
	1391	+ return;
	1392	+}
	1393	+
	1394	+static void trace_imc_event_del(struct perf_event *event, int flags)
	1395	+{
	1396	+ int core_id = smp_processor_id() / threads_per_core;
	1397	+ struct imc_pmu_ref *ref = NULL;
	1398	+
	1399	+ if (trace_imc_refc)
	1400	+ ref = &trace_imc_refc[core_id];
	1401	+ if (!ref) {
	1402	+ pr_debug("imc: Failed to get event reference count\n");
	1403	+ return;
	1404	+ }
	1405	+
	1406	+ spin_lock(&ref->lock);
	1407	+ ref->refc--;
	1408	+ if (ref->refc == 0) {
	1409	+ if (opal_imc_counters_stop(OPAL_IMC_COUNTERS_TRACE,
	1410	+ get_hard_smp_processor_id(smp_processor_id()))) {
	1411	+ spin_unlock(&ref->lock);
	1412	+ pr_err("trace-imc: Unable to stop the counters for core %d\n", core_id);
	1413	+ return;
	1414	+ }
	1415	+ } else if (ref->refc < 0) {
	1416	+ ref->refc = 0;
	1417	+ }
	1418	+ spin_unlock(&ref->lock);
	1419	+
	1420	+ trace_imc_event_stop(event, flags);
	1421	+}
	1422	+
	1423	+static int trace_imc_event_init(struct perf_event *event)
	1424	+{
	1425	+ if (event->attr.type != event->pmu->type)
	1426	+ return -ENOENT;
	1427	+
	1428	+ if (!perfmon_capable())
	1429	+ return -EACCES;
	1430	+
	1431	+ /* Return if this is a couting event */
	1432	+ if (event->attr.sample_period == 0)
	1433	+ return -ENOENT;
	1434	+
	1435	+ /*
	1436	+ * Take the global lock, and make sure
	1437	+ * no other thread is running any core/thread imc
	1438	+ * events
	1439	+ */
	1440	+ spin_lock(&imc_global_refc.lock);
	1441	+ if (imc_global_refc.id == 0 \|\| imc_global_refc.id == IMC_DOMAIN_TRACE) {
	1442	+ /*
	1443	+ * No core/thread imc events are running in the
	1444	+ * system, so set the refc.id to trace-imc.
	1445	+ */
	1446	+ imc_global_refc.id = IMC_DOMAIN_TRACE;
	1447	+ imc_global_refc.refc++;
	1448	+ } else {
	1449	+ spin_unlock(&imc_global_refc.lock);
	1450	+ return -EBUSY;
	1451	+ }
	1452	+ spin_unlock(&imc_global_refc.lock);
	1453	+
	1454	+ event->hw.idx = -1;
	1455	+
	1456	+ /*
	1457	+ * There can only be a single PMU for perf_hw_context events which is assigned to
	1458	+ * core PMU. Hence use "perf_sw_context" for trace_imc.
	1459	+ */
	1460	+ event->pmu->task_ctx_nr = perf_sw_context;
	1461	+ event->destroy = reset_global_refc;
	1462	+ return 0;
1065	1463	}
1066	1464
1067	1465	/* update_pmu_ops : Populate the appropriate operations for "pmu" */
..	..	@@ -1074,6 +1472,7 @@
1074	1472	pmu->pmu.stop = imc_event_stop;
1075	1473	pmu->pmu.read = imc_event_update;
1076	1474	pmu->pmu.attr_groups = pmu->attr_groups;
	1475	+ pmu->pmu.capabilities = PERF_PMU_CAP_NO_EXCLUDE;
1077	1476	pmu->attr_groups[IMC_FORMAT_ATTR] = &imc_format_group;
1078	1477
1079	1478	switch (pmu->domain) {
..	..	@@ -1093,6 +1492,14 @@
1093	1492	pmu->pmu.cancel_txn = thread_imc_pmu_cancel_txn;
1094	1493	pmu->pmu.commit_txn = thread_imc_pmu_commit_txn;
1095	1494	break;
	1495	+ case IMC_DOMAIN_TRACE:
	1496	+ pmu->pmu.event_init = trace_imc_event_init;
	1497	+ pmu->pmu.add = trace_imc_event_add;
	1498	+ pmu->pmu.del = trace_imc_event_del;
	1499	+ pmu->pmu.start = trace_imc_event_start;
	1500	+ pmu->pmu.stop = trace_imc_event_stop;
	1501	+ pmu->pmu.read = trace_imc_event_read;
	1502	+ pmu->attr_groups[IMC_FORMAT_ATTR] = &trace_imc_format_group;
1096	1503	default:
1097	1504	break;
1098	1505	}
..	..	@@ -1114,10 +1521,10 @@
1114	1521	i = 0;
1115	1522	for_each_node(nid) {
1116	1523	/*
1117		- * Mutex lock to avoid races while tracking the number of
	1524	+ * Take the lock to avoid races while tracking the number of
1118	1525	* sessions using the chip's nest pmu units.
1119	1526	*/
1120		- mutex_init(&nest_imc_refc[i].lock);
	1527	+ spin_lock_init(&nest_imc_refc[i].lock);
1121	1528
1122	1529	/*
1123	1530	* Loop to init the "id" with the node_id. Variable "i" initialized to
..	..	@@ -1163,10 +1570,10 @@
1163	1570	static void thread_imc_ldbar_disable(void *dummy)
1164	1571	{
1165	1572	/*
1166		- * By Zeroing LDBAR, we disable thread-imc
1167		- * updates.
	1573	+ * By setting 0th bit of LDBAR to zero, we disable thread-imc
	1574	+ * updates to memory.
1168	1575	*/
1169		- mtspr(SPRN_LDBAR, 0);
	1576	+ mtspr(SPRN_LDBAR, (mfspr(SPRN_LDBAR) & (~(1UL << 63))));
1170	1577	}
1171	1578
1172	1579	void thread_imc_disable(void)
..	..	@@ -1183,6 +1590,18 @@
1183	1590	free_pages((u64)per_cpu(thread_imc_mem, i), order);
1184	1591
1185	1592	}
	1593	+}
	1594	+
	1595	+static void cleanup_all_trace_imc_memory(void)
	1596	+{
	1597	+ int i, order = get_order(trace_imc_mem_size);
	1598	+
	1599	+ for_each_online_cpu(i) {
	1600	+ if (per_cpu(trace_imc_mem, i))
	1601	+ free_pages((u64)per_cpu(trace_imc_mem, i), order);
	1602	+
	1603	+ }
	1604	+ kfree(trace_imc_refc);
1186	1605	}
1187	1606
1188	1607	/* Function to free the attr_groups which are dynamically allocated */
..	..	@@ -1225,6 +1644,11 @@
1225	1644	if (pmu_ptr->domain == IMC_DOMAIN_THREAD) {
1226	1645	cpuhp_remove_state(CPUHP_AP_PERF_POWERPC_THREAD_IMC_ONLINE);
1227	1646	cleanup_all_thread_imc_memory();
	1647	+ }
	1648	+
	1649	+ if (pmu_ptr->domain == IMC_DOMAIN_TRACE) {
	1650	+ cpuhp_remove_state(CPUHP_AP_PERF_POWERPC_TRACE_IMC_ONLINE);
	1651	+ cleanup_all_trace_imc_memory();
1228	1652	}
1229	1653	}
1230	1654
..	..	@@ -1308,6 +1732,27 @@
1308	1732
1309	1733	thread_imc_pmu = pmu_ptr;
1310	1734	break;
	1735	+ case IMC_DOMAIN_TRACE:
	1736	+ /* Update the pmu name */
	1737	+ pmu_ptr->pmu.name = kasprintf(GFP_KERNEL, "%s%s", s, "_imc");
	1738	+ if (!pmu_ptr->pmu.name)
	1739	+ return -ENOMEM;
	1740	+
	1741	+ nr_cores = DIV_ROUND_UP(num_possible_cpus(), threads_per_core);
	1742	+ trace_imc_refc = kcalloc(nr_cores, sizeof(struct imc_pmu_ref),
	1743	+ GFP_KERNEL);
	1744	+ if (!trace_imc_refc)
	1745	+ return -ENOMEM;
	1746	+
	1747	+ trace_imc_mem_size = pmu_ptr->counter_mem_size;
	1748	+ for_each_online_cpu(cpu) {
	1749	+ res = trace_imc_mem_alloc(cpu, trace_imc_mem_size);
	1750	+ if (res) {
	1751	+ cleanup_all_trace_imc_memory();
	1752	+ goto err;
	1753	+ }
	1754	+ }
	1755	+ break;
1311	1756	default:
1312	1757	return -EINVAL;
1313	1758	}
..	..	@@ -1381,6 +1826,14 @@
1381	1826	}
1382	1827
1383	1828	break;
	1829	+ case IMC_DOMAIN_TRACE:
	1830	+ ret = trace_imc_cpu_init();
	1831	+ if (ret) {
	1832	+ cleanup_all_trace_imc_memory();
	1833	+ goto err_free_mem;
	1834	+ }
	1835	+
	1836	+ break;
1384	1837	default:
1385	1838	return -EINVAL; /* Unknown domain */
1386	1839	}
..	..	@@ -1397,7 +1850,7 @@
1397	1850	if (ret)
1398	1851	goto err_free_cpuhp_mem;
1399	1852
1400		- pr_info("%s performance monitor hardware support registered\n",
	1853	+ pr_debug("%s performance monitor hardware support registered\n",
1401	1854	pmu_ptr->pmu.name);
1402	1855
1403	1856	return 0;