~hc/RK356X_SDK_RELEASE.git

..	..	@@ -28,6 +28,8 @@
28	28	#include "kfd_pm4_headers_vi.h"
29	29	#include "cwsr_trap_handler.h"
30	30	#include "kfd_iommu.h"
	31	+#include "amdgpu_amdkfd.h"
	32	+#include "kfd_smi_events.h"
31	33
32	34	#define MQD_SIZE_ALIGNED 768
33	35
..	..	@@ -38,9 +40,48 @@
38	40	*/
39	41	static atomic_t kfd_locked = ATOMIC_INIT(0);
40	42
	43	+#ifdef CONFIG_DRM_AMDGPU_CIK
	44	+extern const struct kfd2kgd_calls gfx_v7_kfd2kgd;
	45	+#endif
	46	+extern const struct kfd2kgd_calls gfx_v8_kfd2kgd;
	47	+extern const struct kfd2kgd_calls gfx_v9_kfd2kgd;
	48	+extern const struct kfd2kgd_calls arcturus_kfd2kgd;
	49	+extern const struct kfd2kgd_calls gfx_v10_kfd2kgd;
	50	+extern const struct kfd2kgd_calls gfx_v10_3_kfd2kgd;
	51	+
	52	+static const struct kfd2kgd_calls *kfd2kgd_funcs[] = {
	53	+#ifdef KFD_SUPPORT_IOMMU_V2
	54	+#ifdef CONFIG_DRM_AMDGPU_CIK
	55	+ [CHIP_KAVERI] = &gfx_v7_kfd2kgd,
	56	+#endif
	57	+ [CHIP_CARRIZO] = &gfx_v8_kfd2kgd,
	58	+ [CHIP_RAVEN] = &gfx_v9_kfd2kgd,
	59	+#endif
	60	+#ifdef CONFIG_DRM_AMDGPU_CIK
	61	+ [CHIP_HAWAII] = &gfx_v7_kfd2kgd,
	62	+#endif
	63	+ [CHIP_TONGA] = &gfx_v8_kfd2kgd,
	64	+ [CHIP_FIJI] = &gfx_v8_kfd2kgd,
	65	+ [CHIP_POLARIS10] = &gfx_v8_kfd2kgd,
	66	+ [CHIP_POLARIS11] = &gfx_v8_kfd2kgd,
	67	+ [CHIP_POLARIS12] = &gfx_v8_kfd2kgd,
	68	+ [CHIP_VEGAM] = &gfx_v8_kfd2kgd,
	69	+ [CHIP_VEGA10] = &gfx_v9_kfd2kgd,
	70	+ [CHIP_VEGA12] = &gfx_v9_kfd2kgd,
	71	+ [CHIP_VEGA20] = &gfx_v9_kfd2kgd,
	72	+ [CHIP_RENOIR] = &gfx_v9_kfd2kgd,
	73	+ [CHIP_ARCTURUS] = &arcturus_kfd2kgd,
	74	+ [CHIP_NAVI10] = &gfx_v10_kfd2kgd,
	75	+ [CHIP_NAVI12] = &gfx_v10_kfd2kgd,
	76	+ [CHIP_NAVI14] = &gfx_v10_kfd2kgd,
	77	+ [CHIP_SIENNA_CICHLID] = &gfx_v10_3_kfd2kgd,
	78	+ [CHIP_NAVY_FLOUNDER] = &gfx_v10_3_kfd2kgd,
	79	+};
	80	+
41	81	#ifdef KFD_SUPPORT_IOMMU_V2
42	82	static const struct kfd_device_info kaveri_device_info = {
43	83	.asic_family = CHIP_KAVERI,
	84	+ .asic_name = "kaveri",
44	85	.max_pasid_bits = 16,
45	86	/* max num of queues for KV.TODO should be a dynamic value */
46	87	.max_no_of_hqd = 24,
..	..	@@ -53,10 +94,13 @@
53	94	.needs_iommu_device = true,
54	95	.needs_pci_atomics = false,
55	96	.num_sdma_engines = 2,
	97	+ .num_xgmi_sdma_engines = 0,
	98	+ .num_sdma_queues_per_engine = 2,
56	99	};
57	100
58	101	static const struct kfd_device_info carrizo_device_info = {
59	102	.asic_family = CHIP_CARRIZO,
	103	+ .asic_name = "carrizo",
60	104	.max_pasid_bits = 16,
61	105	/* max num of queues for CZ.TODO should be a dynamic value */
62	106	.max_no_of_hqd = 24,
..	..	@@ -69,10 +113,14 @@
69	113	.needs_iommu_device = true,
70	114	.needs_pci_atomics = false,
71	115	.num_sdma_engines = 2,
	116	+ .num_xgmi_sdma_engines = 0,
	117	+ .num_sdma_queues_per_engine = 2,
72	118	};
	119	+#endif
73	120
74	121	static const struct kfd_device_info raven_device_info = {
75	122	.asic_family = CHIP_RAVEN,
	123	+ .asic_name = "raven",
76	124	.max_pasid_bits = 16,
77	125	.max_no_of_hqd = 24,
78	126	.doorbell_size = 8,
..	..	@@ -84,11 +132,13 @@
84	132	.needs_iommu_device = true,
85	133	.needs_pci_atomics = true,
86	134	.num_sdma_engines = 1,
	135	+ .num_xgmi_sdma_engines = 0,
	136	+ .num_sdma_queues_per_engine = 2,
87	137	};
88		-#endif
89	138
90	139	static const struct kfd_device_info hawaii_device_info = {
91	140	.asic_family = CHIP_HAWAII,
	141	+ .asic_name = "hawaii",
92	142	.max_pasid_bits = 16,
93	143	/* max num of queues for KV.TODO should be a dynamic value */
94	144	.max_no_of_hqd = 24,
..	..	@@ -101,10 +151,13 @@
101	151	.needs_iommu_device = false,
102	152	.needs_pci_atomics = false,
103	153	.num_sdma_engines = 2,
	154	+ .num_xgmi_sdma_engines = 0,
	155	+ .num_sdma_queues_per_engine = 2,
104	156	};
105	157
106	158	static const struct kfd_device_info tonga_device_info = {
107	159	.asic_family = CHIP_TONGA,
	160	+ .asic_name = "tonga",
108	161	.max_pasid_bits = 16,
109	162	.max_no_of_hqd = 24,
110	163	.doorbell_size = 4,
..	..	@@ -116,25 +169,13 @@
116	169	.needs_iommu_device = false,
117	170	.needs_pci_atomics = true,
118	171	.num_sdma_engines = 2,
119		-};
120		-
121		-static const struct kfd_device_info tonga_vf_device_info = {
122		- .asic_family = CHIP_TONGA,
123		- .max_pasid_bits = 16,
124		- .max_no_of_hqd = 24,
125		- .doorbell_size = 4,
126		- .ih_ring_entry_size = 4 * sizeof(uint32_t),
127		- .event_interrupt_class = &event_interrupt_class_cik,
128		- .num_of_watch_points = 4,
129		- .mqd_size_aligned = MQD_SIZE_ALIGNED,
130		- .supports_cwsr = false,
131		- .needs_iommu_device = false,
132		- .needs_pci_atomics = false,
133		- .num_sdma_engines = 2,
	172	+ .num_xgmi_sdma_engines = 0,
	173	+ .num_sdma_queues_per_engine = 2,
134	174	};
135	175
136	176	static const struct kfd_device_info fiji_device_info = {
137	177	.asic_family = CHIP_FIJI,
	178	+ .asic_name = "fiji",
138	179	.max_pasid_bits = 16,
139	180	.max_no_of_hqd = 24,
140	181	.doorbell_size = 4,
..	..	@@ -146,10 +187,13 @@
146	187	.needs_iommu_device = false,
147	188	.needs_pci_atomics = true,
148	189	.num_sdma_engines = 2,
	190	+ .num_xgmi_sdma_engines = 0,
	191	+ .num_sdma_queues_per_engine = 2,
149	192	};
150	193
151	194	static const struct kfd_device_info fiji_vf_device_info = {
152	195	.asic_family = CHIP_FIJI,
	196	+ .asic_name = "fiji",
153	197	.max_pasid_bits = 16,
154	198	.max_no_of_hqd = 24,
155	199	.doorbell_size = 4,
..	..	@@ -161,11 +205,14 @@
161	205	.needs_iommu_device = false,
162	206	.needs_pci_atomics = false,
163	207	.num_sdma_engines = 2,
	208	+ .num_xgmi_sdma_engines = 0,
	209	+ .num_sdma_queues_per_engine = 2,
164	210	};
165	211
166	212
167	213	static const struct kfd_device_info polaris10_device_info = {
168	214	.asic_family = CHIP_POLARIS10,
	215	+ .asic_name = "polaris10",
169	216	.max_pasid_bits = 16,
170	217	.max_no_of_hqd = 24,
171	218	.doorbell_size = 4,
..	..	@@ -177,10 +224,13 @@
177	224	.needs_iommu_device = false,
178	225	.needs_pci_atomics = true,
179	226	.num_sdma_engines = 2,
	227	+ .num_xgmi_sdma_engines = 0,
	228	+ .num_sdma_queues_per_engine = 2,
180	229	};
181	230
182	231	static const struct kfd_device_info polaris10_vf_device_info = {
183	232	.asic_family = CHIP_POLARIS10,
	233	+ .asic_name = "polaris10",
184	234	.max_pasid_bits = 16,
185	235	.max_no_of_hqd = 24,
186	236	.doorbell_size = 4,
..	..	@@ -192,10 +242,13 @@
192	242	.needs_iommu_device = false,
193	243	.needs_pci_atomics = false,
194	244	.num_sdma_engines = 2,
	245	+ .num_xgmi_sdma_engines = 0,
	246	+ .num_sdma_queues_per_engine = 2,
195	247	};
196	248
197	249	static const struct kfd_device_info polaris11_device_info = {
198	250	.asic_family = CHIP_POLARIS11,
	251	+ .asic_name = "polaris11",
199	252	.max_pasid_bits = 16,
200	253	.max_no_of_hqd = 24,
201	254	.doorbell_size = 4,
..	..	@@ -207,10 +260,49 @@
207	260	.needs_iommu_device = false,
208	261	.needs_pci_atomics = true,
209	262	.num_sdma_engines = 2,
	263	+ .num_xgmi_sdma_engines = 0,
	264	+ .num_sdma_queues_per_engine = 2,
	265	+};
	266	+
	267	+static const struct kfd_device_info polaris12_device_info = {
	268	+ .asic_family = CHIP_POLARIS12,
	269	+ .asic_name = "polaris12",
	270	+ .max_pasid_bits = 16,
	271	+ .max_no_of_hqd = 24,
	272	+ .doorbell_size = 4,
	273	+ .ih_ring_entry_size = 4 * sizeof(uint32_t),
	274	+ .event_interrupt_class = &event_interrupt_class_cik,
	275	+ .num_of_watch_points = 4,
	276	+ .mqd_size_aligned = MQD_SIZE_ALIGNED,
	277	+ .supports_cwsr = true,
	278	+ .needs_iommu_device = false,
	279	+ .needs_pci_atomics = true,
	280	+ .num_sdma_engines = 2,
	281	+ .num_xgmi_sdma_engines = 0,
	282	+ .num_sdma_queues_per_engine = 2,
	283	+};
	284	+
	285	+static const struct kfd_device_info vegam_device_info = {
	286	+ .asic_family = CHIP_VEGAM,
	287	+ .asic_name = "vegam",
	288	+ .max_pasid_bits = 16,
	289	+ .max_no_of_hqd = 24,
	290	+ .doorbell_size = 4,
	291	+ .ih_ring_entry_size = 4 * sizeof(uint32_t),
	292	+ .event_interrupt_class = &event_interrupt_class_cik,
	293	+ .num_of_watch_points = 4,
	294	+ .mqd_size_aligned = MQD_SIZE_ALIGNED,
	295	+ .supports_cwsr = true,
	296	+ .needs_iommu_device = false,
	297	+ .needs_pci_atomics = true,
	298	+ .num_sdma_engines = 2,
	299	+ .num_xgmi_sdma_engines = 0,
	300	+ .num_sdma_queues_per_engine = 2,
210	301	};
211	302
212	303	static const struct kfd_device_info vega10_device_info = {
213	304	.asic_family = CHIP_VEGA10,
	305	+ .asic_name = "vega10",
214	306	.max_pasid_bits = 16,
215	307	.max_no_of_hqd = 24,
216	308	.doorbell_size = 8,
..	..	@@ -222,10 +314,13 @@
222	314	.needs_iommu_device = false,
223	315	.needs_pci_atomics = false,
224	316	.num_sdma_engines = 2,
	317	+ .num_xgmi_sdma_engines = 0,
	318	+ .num_sdma_queues_per_engine = 2,
225	319	};
226	320
227	321	static const struct kfd_device_info vega10_vf_device_info = {
228	322	.asic_family = CHIP_VEGA10,
	323	+ .asic_name = "vega10",
229	324	.max_pasid_bits = 16,
230	325	.max_no_of_hqd = 24,
231	326	.doorbell_size = 8,
..	..	@@ -237,104 +332,196 @@
237	332	.needs_iommu_device = false,
238	333	.needs_pci_atomics = false,
239	334	.num_sdma_engines = 2,
	335	+ .num_xgmi_sdma_engines = 0,
	336	+ .num_sdma_queues_per_engine = 2,
240	337	};
241	338
242		-
243		-struct kfd_deviceid {
244		- unsigned short did;
245		- const struct kfd_device_info *device_info;
	339	+static const struct kfd_device_info vega12_device_info = {
	340	+ .asic_family = CHIP_VEGA12,
	341	+ .asic_name = "vega12",
	342	+ .max_pasid_bits = 16,
	343	+ .max_no_of_hqd = 24,
	344	+ .doorbell_size = 8,
	345	+ .ih_ring_entry_size = 8 * sizeof(uint32_t),
	346	+ .event_interrupt_class = &event_interrupt_class_v9,
	347	+ .num_of_watch_points = 4,
	348	+ .mqd_size_aligned = MQD_SIZE_ALIGNED,
	349	+ .supports_cwsr = true,
	350	+ .needs_iommu_device = false,
	351	+ .needs_pci_atomics = false,
	352	+ .num_sdma_engines = 2,
	353	+ .num_xgmi_sdma_engines = 0,
	354	+ .num_sdma_queues_per_engine = 2,
246	355	};
247	356
248		-static const struct kfd_deviceid supported_devices[] = {
	357	+static const struct kfd_device_info vega20_device_info = {
	358	+ .asic_family = CHIP_VEGA20,
	359	+ .asic_name = "vega20",
	360	+ .max_pasid_bits = 16,
	361	+ .max_no_of_hqd = 24,
	362	+ .doorbell_size = 8,
	363	+ .ih_ring_entry_size = 8 * sizeof(uint32_t),
	364	+ .event_interrupt_class = &event_interrupt_class_v9,
	365	+ .num_of_watch_points = 4,
	366	+ .mqd_size_aligned = MQD_SIZE_ALIGNED,
	367	+ .supports_cwsr = true,
	368	+ .needs_iommu_device = false,
	369	+ .needs_pci_atomics = false,
	370	+ .num_sdma_engines = 2,
	371	+ .num_xgmi_sdma_engines = 0,
	372	+ .num_sdma_queues_per_engine = 8,
	373	+};
	374	+
	375	+static const struct kfd_device_info arcturus_device_info = {
	376	+ .asic_family = CHIP_ARCTURUS,
	377	+ .asic_name = "arcturus",
	378	+ .max_pasid_bits = 16,
	379	+ .max_no_of_hqd = 24,
	380	+ .doorbell_size = 8,
	381	+ .ih_ring_entry_size = 8 * sizeof(uint32_t),
	382	+ .event_interrupt_class = &event_interrupt_class_v9,
	383	+ .num_of_watch_points = 4,
	384	+ .mqd_size_aligned = MQD_SIZE_ALIGNED,
	385	+ .supports_cwsr = true,
	386	+ .needs_iommu_device = false,
	387	+ .needs_pci_atomics = false,
	388	+ .num_sdma_engines = 2,
	389	+ .num_xgmi_sdma_engines = 6,
	390	+ .num_sdma_queues_per_engine = 8,
	391	+};
	392	+
	393	+static const struct kfd_device_info renoir_device_info = {
	394	+ .asic_family = CHIP_RENOIR,
	395	+ .asic_name = "renoir",
	396	+ .max_pasid_bits = 16,
	397	+ .max_no_of_hqd = 24,
	398	+ .doorbell_size = 8,
	399	+ .ih_ring_entry_size = 8 * sizeof(uint32_t),
	400	+ .event_interrupt_class = &event_interrupt_class_v9,
	401	+ .num_of_watch_points = 4,
	402	+ .mqd_size_aligned = MQD_SIZE_ALIGNED,
	403	+ .supports_cwsr = true,
	404	+ .needs_iommu_device = false,
	405	+ .needs_pci_atomics = false,
	406	+ .num_sdma_engines = 1,
	407	+ .num_xgmi_sdma_engines = 0,
	408	+ .num_sdma_queues_per_engine = 2,
	409	+};
	410	+
	411	+static const struct kfd_device_info navi10_device_info = {
	412	+ .asic_family = CHIP_NAVI10,
	413	+ .asic_name = "navi10",
	414	+ .max_pasid_bits = 16,
	415	+ .max_no_of_hqd = 24,
	416	+ .doorbell_size = 8,
	417	+ .ih_ring_entry_size = 8 * sizeof(uint32_t),
	418	+ .event_interrupt_class = &event_interrupt_class_v9,
	419	+ .num_of_watch_points = 4,
	420	+ .mqd_size_aligned = MQD_SIZE_ALIGNED,
	421	+ .needs_iommu_device = false,
	422	+ .supports_cwsr = true,
	423	+ .needs_pci_atomics = false,
	424	+ .num_sdma_engines = 2,
	425	+ .num_xgmi_sdma_engines = 0,
	426	+ .num_sdma_queues_per_engine = 8,
	427	+};
	428	+
	429	+static const struct kfd_device_info navi12_device_info = {
	430	+ .asic_family = CHIP_NAVI12,
	431	+ .asic_name = "navi12",
	432	+ .max_pasid_bits = 16,
	433	+ .max_no_of_hqd = 24,
	434	+ .doorbell_size = 8,
	435	+ .ih_ring_entry_size = 8 * sizeof(uint32_t),
	436	+ .event_interrupt_class = &event_interrupt_class_v9,
	437	+ .num_of_watch_points = 4,
	438	+ .mqd_size_aligned = MQD_SIZE_ALIGNED,
	439	+ .needs_iommu_device = false,
	440	+ .supports_cwsr = true,
	441	+ .needs_pci_atomics = false,
	442	+ .num_sdma_engines = 2,
	443	+ .num_xgmi_sdma_engines = 0,
	444	+ .num_sdma_queues_per_engine = 8,
	445	+};
	446	+
	447	+static const struct kfd_device_info navi14_device_info = {
	448	+ .asic_family = CHIP_NAVI14,
	449	+ .asic_name = "navi14",
	450	+ .max_pasid_bits = 16,
	451	+ .max_no_of_hqd = 24,
	452	+ .doorbell_size = 8,
	453	+ .ih_ring_entry_size = 8 * sizeof(uint32_t),
	454	+ .event_interrupt_class = &event_interrupt_class_v9,
	455	+ .num_of_watch_points = 4,
	456	+ .mqd_size_aligned = MQD_SIZE_ALIGNED,
	457	+ .needs_iommu_device = false,
	458	+ .supports_cwsr = true,
	459	+ .needs_pci_atomics = false,
	460	+ .num_sdma_engines = 2,
	461	+ .num_xgmi_sdma_engines = 0,
	462	+ .num_sdma_queues_per_engine = 8,
	463	+};
	464	+
	465	+static const struct kfd_device_info sienna_cichlid_device_info = {
	466	+ .asic_family = CHIP_SIENNA_CICHLID,
	467	+ .asic_name = "sienna_cichlid",
	468	+ .max_pasid_bits = 16,
	469	+ .max_no_of_hqd = 24,
	470	+ .doorbell_size = 8,
	471	+ .ih_ring_entry_size = 8 * sizeof(uint32_t),
	472	+ .event_interrupt_class = &event_interrupt_class_v9,
	473	+ .num_of_watch_points = 4,
	474	+ .mqd_size_aligned = MQD_SIZE_ALIGNED,
	475	+ .needs_iommu_device = false,
	476	+ .supports_cwsr = true,
	477	+ .needs_pci_atomics = false,
	478	+ .num_sdma_engines = 4,
	479	+ .num_xgmi_sdma_engines = 0,
	480	+ .num_sdma_queues_per_engine = 8,
	481	+};
	482	+
	483	+static const struct kfd_device_info navy_flounder_device_info = {
	484	+ .asic_family = CHIP_NAVY_FLOUNDER,
	485	+ .asic_name = "navy_flounder",
	486	+ .max_pasid_bits = 16,
	487	+ .max_no_of_hqd = 24,
	488	+ .doorbell_size = 8,
	489	+ .ih_ring_entry_size = 8 * sizeof(uint32_t),
	490	+ .event_interrupt_class = &event_interrupt_class_v9,
	491	+ .num_of_watch_points = 4,
	492	+ .mqd_size_aligned = MQD_SIZE_ALIGNED,
	493	+ .needs_iommu_device = false,
	494	+ .supports_cwsr = true,
	495	+ .needs_pci_atomics = false,
	496	+ .num_sdma_engines = 2,
	497	+ .num_xgmi_sdma_engines = 0,
	498	+ .num_sdma_queues_per_engine = 8,
	499	+};
	500	+
	501	+/* For each entry, [0] is regular and [1] is virtualisation device. */
	502	+static const struct kfd_device_info *kfd_supported_devices[][2] = {
249	503	#ifdef KFD_SUPPORT_IOMMU_V2
250		- { 0x1304, &kaveri_device_info }, /* Kaveri */
251		- { 0x1305, &kaveri_device_info }, /* Kaveri */
252		- { 0x1306, &kaveri_device_info }, /* Kaveri */
253		- { 0x1307, &kaveri_device_info }, /* Kaveri */
254		- { 0x1309, &kaveri_device_info }, /* Kaveri */
255		- { 0x130A, &kaveri_device_info }, /* Kaveri */
256		- { 0x130B, &kaveri_device_info }, /* Kaveri */
257		- { 0x130C, &kaveri_device_info }, /* Kaveri */
258		- { 0x130D, &kaveri_device_info }, /* Kaveri */
259		- { 0x130E, &kaveri_device_info }, /* Kaveri */
260		- { 0x130F, &kaveri_device_info }, /* Kaveri */
261		- { 0x1310, &kaveri_device_info }, /* Kaveri */
262		- { 0x1311, &kaveri_device_info }, /* Kaveri */
263		- { 0x1312, &kaveri_device_info }, /* Kaveri */
264		- { 0x1313, &kaveri_device_info }, /* Kaveri */
265		- { 0x1315, &kaveri_device_info }, /* Kaveri */
266		- { 0x1316, &kaveri_device_info }, /* Kaveri */
267		- { 0x1317, &kaveri_device_info }, /* Kaveri */
268		- { 0x1318, &kaveri_device_info }, /* Kaveri */
269		- { 0x131B, &kaveri_device_info }, /* Kaveri */
270		- { 0x131C, &kaveri_device_info }, /* Kaveri */
271		- { 0x131D, &kaveri_device_info }, /* Kaveri */
272		- { 0x9870, &carrizo_device_info }, /* Carrizo */
273		- { 0x9874, &carrizo_device_info }, /* Carrizo */
274		- { 0x9875, &carrizo_device_info }, /* Carrizo */
275		- { 0x9876, &carrizo_device_info }, /* Carrizo */
276		- { 0x9877, &carrizo_device_info }, /* Carrizo */
277		- { 0x15DD, &raven_device_info }, /* Raven */
	504	+ [CHIP_KAVERI] = {&kaveri_device_info, NULL},
	505	+ [CHIP_CARRIZO] = {&carrizo_device_info, NULL},
278	506	#endif
279		- { 0x67A0, &hawaii_device_info }, /* Hawaii */
280		- { 0x67A1, &hawaii_device_info }, /* Hawaii */
281		- { 0x67A2, &hawaii_device_info }, /* Hawaii */
282		- { 0x67A8, &hawaii_device_info }, /* Hawaii */
283		- { 0x67A9, &hawaii_device_info }, /* Hawaii */
284		- { 0x67AA, &hawaii_device_info }, /* Hawaii */
285		- { 0x67B0, &hawaii_device_info }, /* Hawaii */
286		- { 0x67B1, &hawaii_device_info }, /* Hawaii */
287		- { 0x67B8, &hawaii_device_info }, /* Hawaii */
288		- { 0x67B9, &hawaii_device_info }, /* Hawaii */
289		- { 0x67BA, &hawaii_device_info }, /* Hawaii */
290		- { 0x67BE, &hawaii_device_info }, /* Hawaii */
291		- { 0x6920, &tonga_device_info }, /* Tonga */
292		- { 0x6921, &tonga_device_info }, /* Tonga */
293		- { 0x6928, &tonga_device_info }, /* Tonga */
294		- { 0x6929, &tonga_device_info }, /* Tonga */
295		- { 0x692B, &tonga_device_info }, /* Tonga */
296		- { 0x692F, &tonga_vf_device_info }, /* Tonga vf */
297		- { 0x6938, &tonga_device_info }, /* Tonga */
298		- { 0x6939, &tonga_device_info }, /* Tonga */
299		- { 0x7300, &fiji_device_info }, /* Fiji */
300		- { 0x730F, &fiji_vf_device_info }, /* Fiji vf*/
301		- { 0x67C0, &polaris10_device_info }, /* Polaris10 */
302		- { 0x67C1, &polaris10_device_info }, /* Polaris10 */
303		- { 0x67C2, &polaris10_device_info }, /* Polaris10 */
304		- { 0x67C4, &polaris10_device_info }, /* Polaris10 */
305		- { 0x67C7, &polaris10_device_info }, /* Polaris10 */
306		- { 0x67C8, &polaris10_device_info }, /* Polaris10 */
307		- { 0x67C9, &polaris10_device_info }, /* Polaris10 */
308		- { 0x67CA, &polaris10_device_info }, /* Polaris10 */
309		- { 0x67CC, &polaris10_device_info }, /* Polaris10 */
310		- { 0x67CF, &polaris10_device_info }, /* Polaris10 */
311		- { 0x67D0, &polaris10_vf_device_info }, /* Polaris10 vf*/
312		- { 0x67DF, &polaris10_device_info }, /* Polaris10 */
313		- { 0x6FDF, &polaris10_device_info }, /* Polaris10 */
314		- { 0x67E0, &polaris11_device_info }, /* Polaris11 */
315		- { 0x67E1, &polaris11_device_info }, /* Polaris11 */
316		- { 0x67E3, &polaris11_device_info }, /* Polaris11 */
317		- { 0x67E7, &polaris11_device_info }, /* Polaris11 */
318		- { 0x67E8, &polaris11_device_info }, /* Polaris11 */
319		- { 0x67E9, &polaris11_device_info }, /* Polaris11 */
320		- { 0x67EB, &polaris11_device_info }, /* Polaris11 */
321		- { 0x67EF, &polaris11_device_info }, /* Polaris11 */
322		- { 0x67FF, &polaris11_device_info }, /* Polaris11 */
323		- { 0x6860, &vega10_device_info }, /* Vega10 */
324		- { 0x6861, &vega10_device_info }, /* Vega10 */
325		- { 0x6862, &vega10_device_info }, /* Vega10 */
326		- { 0x6863, &vega10_device_info }, /* Vega10 */
327		- { 0x6864, &vega10_device_info }, /* Vega10 */
328		- { 0x6867, &vega10_device_info }, /* Vega10 */
329		- { 0x6868, &vega10_device_info }, /* Vega10 */
330		- { 0x6869, &vega10_device_info }, /* Vega10 */
331		- { 0x686A, &vega10_device_info }, /* Vega10 */
332		- { 0x686B, &vega10_device_info }, /* Vega10 */
333		- { 0x686C, &vega10_vf_device_info }, /* Vega10 vf*/
334		- { 0x686D, &vega10_device_info }, /* Vega10 */
335		- { 0x686E, &vega10_device_info }, /* Vega10 */
336		- { 0x686F, &vega10_device_info }, /* Vega10 */
337		- { 0x687F, &vega10_device_info }, /* Vega10 */
	507	+ [CHIP_RAVEN] = {&raven_device_info, NULL},
	508	+ [CHIP_HAWAII] = {&hawaii_device_info, NULL},
	509	+ [CHIP_TONGA] = {&tonga_device_info, NULL},
	510	+ [CHIP_FIJI] = {&fiji_device_info, &fiji_vf_device_info},
	511	+ [CHIP_POLARIS10] = {&polaris10_device_info, &polaris10_vf_device_info},
	512	+ [CHIP_POLARIS11] = {&polaris11_device_info, NULL},
	513	+ [CHIP_POLARIS12] = {&polaris12_device_info, NULL},
	514	+ [CHIP_VEGAM] = {&vegam_device_info, NULL},
	515	+ [CHIP_VEGA10] = {&vega10_device_info, &vega10_vf_device_info},
	516	+ [CHIP_VEGA12] = {&vega12_device_info, NULL},
	517	+ [CHIP_VEGA20] = {&vega20_device_info, NULL},
	518	+ [CHIP_RENOIR] = {&renoir_device_info, NULL},
	519	+ [CHIP_ARCTURUS] = {&arcturus_device_info, &arcturus_device_info},
	520	+ [CHIP_NAVI10] = {&navi10_device_info, NULL},
	521	+ [CHIP_NAVI12] = {&navi12_device_info, &navi12_device_info},
	522	+ [CHIP_NAVI14] = {&navi14_device_info, NULL},
	523	+ [CHIP_SIENNA_CICHLID] = {&sienna_cichlid_device_info, &sienna_cichlid_device_info},
	524	+ [CHIP_NAVY_FLOUNDER] = {&navy_flounder_device_info, &navy_flounder_device_info},
338	525	};
339	526
340	527	static int kfd_gtt_sa_init(struct kfd_dev *kfd, unsigned int buf_size,
..	..	@@ -343,47 +530,25 @@
343	530
344	531	static int kfd_resume(struct kfd_dev *kfd);
345	532
346		-static const struct kfd_device_info *lookup_device_info(unsigned short did)
347		-{
348		- size_t i;
349		-
350		- for (i = 0; i < ARRAY_SIZE(supported_devices); i++) {
351		- if (supported_devices[i].did == did) {
352		- WARN_ON(!supported_devices[i].device_info);
353		- return supported_devices[i].device_info;
354		- }
355		- }
356		-
357		- dev_warn(kfd_device, "DID %04x is missing in supported_devices\n",
358		- did);
359		-
360		- return NULL;
361		-}
362		-
363	533	struct kfd_dev kgd2kfd_probe(struct kgd_dev kgd,
364		- struct pci_dev pdev, const struct kfd2kgd_calls f2g)
	534	+ struct pci_dev *pdev, unsigned int asic_type, bool vf)
365	535	{
366	536	struct kfd_dev *kfd;
367		- int ret;
368		- const struct kfd_device_info *device_info =
369		- lookup_device_info(pdev->device);
	537	+ const struct kfd_device_info *device_info;
	538	+ const struct kfd2kgd_calls *f2g;
370	539
371		- if (!device_info) {
372		- dev_err(kfd_device, "kgd2kfd_probe failed\n");
373		- return NULL;
	540	+ if (asic_type >= sizeof(kfd_supported_devices) / (sizeof(void ) 2)
	541	+ \|\| asic_type >= sizeof(kfd2kgd_funcs) / sizeof(void *)) {
	542	+ dev_err(kfd_device, "asic_type %d out of range\n", asic_type);
	543	+ return NULL; /* asic_type out of range */
374	544	}
375	545
376		- /* Allow BIF to recode atomics to PCIe 3.0 AtomicOps.
377		- * 32 and 64-bit requests are possible and must be
378		- * supported.
379		- */
380		- ret = pci_enable_atomic_ops_to_root(pdev,
381		- PCI_EXP_DEVCAP2_ATOMIC_COMP32 \|
382		- PCI_EXP_DEVCAP2_ATOMIC_COMP64);
383		- if (device_info->needs_pci_atomics && ret < 0) {
384		- dev_info(kfd_device,
385		- "skipped device %x:%x, PCI rejects atomics\n",
386		- pdev->vendor, pdev->device);
	546	+ device_info = kfd_supported_devices[asic_type][vf];
	547	+ f2g = kfd2kgd_funcs[asic_type];
	548	+
	549	+ if (!device_info \|\| !f2g) {
	550	+ dev_err(kfd_device, "%s %s not supported in kfd\n",
	551	+ amdgpu_asic_name[asic_type], vf ? "VF" : "");
387	552	return NULL;
388	553	}
389	554
..	..	@@ -391,15 +556,34 @@
391	556	if (!kfd)
392	557	return NULL;
393	558
	559	+ /* Allow BIF to recode atomics to PCIe 3.0 AtomicOps.
	560	+ * 32 and 64-bit requests are possible and must be
	561	+ * supported.
	562	+ */
	563	+ kfd->pci_atomic_requested = amdgpu_amdkfd_have_atomics_support(kgd);
	564	+ if (device_info->needs_pci_atomics &&
	565	+ !kfd->pci_atomic_requested) {
	566	+ dev_info(kfd_device,
	567	+ "skipped device %x:%x, PCI rejects atomics\n",
	568	+ pdev->vendor, pdev->device);
	569	+ kfree(kfd);
	570	+ return NULL;
	571	+ }
	572	+
394	573	kfd->kgd = kgd;
395	574	kfd->device_info = device_info;
396	575	kfd->pdev = pdev;
397	576	kfd->init_complete = false;
398	577	kfd->kfd2kgd = f2g;
	578	+ atomic_set(&kfd->compute_profile, 0);
399	579
400	580	mutex_init(&kfd->doorbell_mutex);
401	581	memset(&kfd->doorbell_available_index, 0,
402	582	sizeof(kfd->doorbell_available_index));
	583	+
	584	+ atomic_set(&kfd->sram_ecc_flag, 0);
	585	+
	586	+ ida_init(&kfd->doorbell_ida);
403	587
404	588	return kfd;
405	589	}
..	..	@@ -411,21 +595,67 @@
411	595	BUILD_BUG_ON(sizeof(cwsr_trap_gfx8_hex) > PAGE_SIZE);
412	596	kfd->cwsr_isa = cwsr_trap_gfx8_hex;
413	597	kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx8_hex);
414		- } else {
	598	+ } else if (kfd->device_info->asic_family == CHIP_ARCTURUS) {
	599	+ BUILD_BUG_ON(sizeof(cwsr_trap_arcturus_hex) > PAGE_SIZE);
	600	+ kfd->cwsr_isa = cwsr_trap_arcturus_hex;
	601	+ kfd->cwsr_isa_size = sizeof(cwsr_trap_arcturus_hex);
	602	+ } else if (kfd->device_info->asic_family < CHIP_NAVI10) {
415	603	BUILD_BUG_ON(sizeof(cwsr_trap_gfx9_hex) > PAGE_SIZE);
416	604	kfd->cwsr_isa = cwsr_trap_gfx9_hex;
417	605	kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx9_hex);
	606	+ } else if (kfd->device_info->asic_family < CHIP_SIENNA_CICHLID) {
	607	+ BUILD_BUG_ON(sizeof(cwsr_trap_nv1x_hex) > PAGE_SIZE);
	608	+ kfd->cwsr_isa = cwsr_trap_nv1x_hex;
	609	+ kfd->cwsr_isa_size = sizeof(cwsr_trap_nv1x_hex);
	610	+ } else {
	611	+ BUILD_BUG_ON(sizeof(cwsr_trap_gfx10_hex) > PAGE_SIZE);
	612	+ kfd->cwsr_isa = cwsr_trap_gfx10_hex;
	613	+ kfd->cwsr_isa_size = sizeof(cwsr_trap_gfx10_hex);
418	614	}
419	615
420	616	kfd->cwsr_enabled = true;
421	617	}
422	618	}
423	619
	620	+static int kfd_gws_init(struct kfd_dev *kfd)
	621	+{
	622	+ int ret = 0;
	623	+
	624	+ if (kfd->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS)
	625	+ return 0;
	626	+
	627	+ if (hws_gws_support
	628	+ \|\| (kfd->device_info->asic_family == CHIP_VEGA10
	629	+ && kfd->mec2_fw_version >= 0x81b3)
	630	+ \|\| (kfd->device_info->asic_family >= CHIP_VEGA12
	631	+ && kfd->device_info->asic_family <= CHIP_RAVEN
	632	+ && kfd->mec2_fw_version >= 0x1b3)
	633	+ \|\| (kfd->device_info->asic_family == CHIP_ARCTURUS
	634	+ && kfd->mec2_fw_version >= 0x30))
	635	+ ret = amdgpu_amdkfd_alloc_gws(kfd->kgd,
	636	+ amdgpu_amdkfd_get_num_gws(kfd->kgd), &kfd->gws);
	637	+
	638	+ return ret;
	639	+}
	640	+
	641	+static void kfd_smi_init(struct kfd_dev *dev) {
	642	+ INIT_LIST_HEAD(&dev->smi_clients);
	643	+ spin_lock_init(&dev->smi_lock);
	644	+}
	645	+
424	646	bool kgd2kfd_device_init(struct kfd_dev *kfd,
	647	+ struct drm_device *ddev,
425	648	const struct kgd2kfd_shared_resources *gpu_resources)
426	649	{
427	650	unsigned int size;
428	651
	652	+ kfd->ddev = ddev;
	653	+ kfd->mec_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd,
	654	+ KGD_ENGINE_MEC1);
	655	+ kfd->mec2_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd,
	656	+ KGD_ENGINE_MEC2);
	657	+ kfd->sdma_fw_version = amdgpu_amdkfd_get_fw_version(kfd->kgd,
	658	+ KGD_ENGINE_SDMA1);
429	659	kfd->shared_resources = *gpu_resources;
430	660
431	661	kfd->vm_info.first_vmid_kfd = ffs(gpu_resources->compute_vmid_bitmap)-1;
..	..	@@ -434,15 +664,10 @@
434	664	- kfd->vm_info.first_vmid_kfd + 1;
435	665
436	666	/* Verify module parameters regarding mapped process number*/
437		- if ((hws_max_conc_proc < 0)
438		- \|\| (hws_max_conc_proc > kfd->vm_info.vmid_num_kfd)) {
439		- dev_err(kfd_device,
440		- "hws_max_conc_proc %d must be between 0 and %d, use %d instead\n",
441		- hws_max_conc_proc, kfd->vm_info.vmid_num_kfd,
442		- kfd->vm_info.vmid_num_kfd);
	667	+ if (hws_max_conc_proc >= 0)
	668	+ kfd->max_proc_per_quantum = min((u32)hws_max_conc_proc, kfd->vm_info.vmid_num_kfd);
	669	+ else
443	670	kfd->max_proc_per_quantum = kfd->vm_info.vmid_num_kfd;
444		- } else
445		- kfd->max_proc_per_quantum = hws_max_conc_proc;
446	671
447	672	/* calculate max size of mqds needed for queues */
448	673	size = max_num_of_queues_per_device *
..	..	@@ -462,12 +687,12 @@
462	687	/* add another 512KB for all other allocations on gart (HPD, fences) */
463	688	size += 512 * 1024;
464	689
465		- if (kfd->kfd2kgd->init_gtt_mem_allocation(
	690	+ if (amdgpu_amdkfd_alloc_gtt_mem(
466	691	kfd->kgd, size, &kfd->gtt_mem,
467	692	&kfd->gtt_start_gpu_addr, &kfd->gtt_start_cpu_ptr,
468	693	false)) {
469	694	dev_err(kfd_device, "Could not allocate %d bytes\n", size);
470		- goto out;
	695	+ goto alloc_gtt_mem_failure;
471	696	}
472	697
473	698	dev_info(kfd_device, "Allocated %d bytes on gart\n", size);
..	..	@@ -484,10 +709,11 @@
484	709	goto kfd_doorbell_error;
485	710	}
486	711
487		- if (kfd_topology_add_device(kfd)) {
488		- dev_err(kfd_device, "Error adding device to topology\n");
489		- goto kfd_topology_add_device_error;
490		- }
	712	+ kfd->hive_id = amdgpu_amdkfd_get_hive_id(kfd->kgd);
	713	+
	714	+ kfd->unique_id = amdgpu_amdkfd_get_unique_id(kfd->kgd);
	715	+
	716	+ kfd->noretry = amdgpu_amdkfd_get_noretry(kfd->kgd);
491	717
492	718	if (kfd_interrupt_init(kfd)) {
493	719	dev_err(kfd_device, "Error initializing interrupts\n");
..	..	@@ -500,17 +726,40 @@
500	726	goto device_queue_manager_error;
501	727	}
502	728
	729	+ /* If supported on this device, allocate global GWS that is shared
	730	+ * by all KFD processes
	731	+ */
	732	+ if (kfd_gws_init(kfd)) {
	733	+ dev_err(kfd_device, "Could not allocate %d gws\n",
	734	+ amdgpu_amdkfd_get_num_gws(kfd->kgd));
	735	+ goto gws_error;
	736	+ }
	737	+
	738	+ /* If CRAT is broken, won't set iommu enabled */
	739	+ kfd_double_confirm_iommu_support(kfd);
	740	+
503	741	if (kfd_iommu_device_init(kfd)) {
	742	+ kfd->use_iommu_v2 = false;
504	743	dev_err(kfd_device, "Error initializing iommuv2\n");
505	744	goto device_iommu_error;
506	745	}
507	746
508	747	kfd_cwsr_init(kfd);
509	748
	749	+ if(kgd2kfd_resume_iommu(kfd))
	750	+ goto device_iommu_error;
	751	+
510	752	if (kfd_resume(kfd))
511	753	goto kfd_resume_error;
512	754
513	755	kfd->dbgmgr = NULL;
	756	+
	757	+ if (kfd_topology_add_device(kfd)) {
	758	+ dev_err(kfd_device, "Error adding device to topology\n");
	759	+ goto kfd_topology_add_device_error;
	760	+ }
	761	+
	762	+ kfd_smi_init(kfd);
514	763
515	764	kfd->init_complete = true;
516	765	dev_info(kfd_device, "added device %x:%x\n", kfd->pdev->vendor,
..	..	@@ -521,19 +770,22 @@
521	770
522	771	goto out;
523	772
	773	+kfd_topology_add_device_error:
524	774	kfd_resume_error:
525	775	device_iommu_error:
	776	+gws_error:
526	777	device_queue_manager_uninit(kfd->dqm);
527	778	device_queue_manager_error:
528	779	kfd_interrupt_exit(kfd);
529	780	kfd_interrupt_error:
530		- kfd_topology_remove_device(kfd);
531		-kfd_topology_add_device_error:
532	781	kfd_doorbell_fini(kfd);
533	782	kfd_doorbell_error:
534	783	kfd_gtt_sa_fini(kfd);
535	784	kfd_gtt_sa_init_error:
536		- kfd->kfd2kgd->free_gtt_mem(kfd->kgd, kfd->gtt_mem);
	785	+ amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem);
	786	+alloc_gtt_mem_failure:
	787	+ if (kfd->gws)
	788	+ amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws);
537	789	dev_err(kfd_device,
538	790	"device %x:%x NOT added due to errors\n",
539	791	kfd->pdev->vendor, kfd->pdev->device);
..	..	@@ -544,13 +796,16 @@
544	796	void kgd2kfd_device_exit(struct kfd_dev *kfd)
545	797	{
546	798	if (kfd->init_complete) {
547		- kgd2kfd_suspend(kfd);
	799	+ kgd2kfd_suspend(kfd, false);
548	800	device_queue_manager_uninit(kfd->dqm);
549	801	kfd_interrupt_exit(kfd);
550	802	kfd_topology_remove_device(kfd);
551	803	kfd_doorbell_fini(kfd);
	804	+ ida_destroy(&kfd->doorbell_ida);
552	805	kfd_gtt_sa_fini(kfd);
553		- kfd->kfd2kgd->free_gtt_mem(kfd->kgd, kfd->gtt_mem);
	806	+ amdgpu_amdkfd_free_gtt_mem(kfd->kgd, kfd->gtt_mem);
	807	+ if (kfd->gws)
	808	+ amdgpu_amdkfd_free_gws(kfd->kgd, kfd->gws);
554	809	}
555	810
556	811	kfree(kfd);
..	..	@@ -560,10 +815,12 @@
560	815	{
561	816	if (!kfd->init_complete)
562	817	return 0;
563		- kgd2kfd_suspend(kfd);
564	818
565		- /* hold dqm->lock to prevent further execution*/
566		- dqm_lock(kfd->dqm);
	819	+ kfd_smi_event_update_gpu_reset(kfd, false);
	820	+
	821	+ kfd->dqm->ops.pre_reset(kfd->dqm);
	822	+
	823	+ kgd2kfd_suspend(kfd, false);
567	824
568	825	kfd_signal_reset_event(kfd);
569	826	return 0;
..	..	@@ -577,18 +834,20 @@
577	834
578	835	int kgd2kfd_post_reset(struct kfd_dev *kfd)
579	836	{
580		- int ret, count;
	837	+ int ret;
581	838
582	839	if (!kfd->init_complete)
583	840	return 0;
584	841
585		- dqm_unlock(kfd->dqm);
586		-
587	842	ret = kfd_resume(kfd);
588	843	if (ret)
589	844	return ret;
590		- count = atomic_dec_return(&kfd_locked);
591		- WARN_ONCE(count != 0, "KFD reset ref. error");
	845	+ atomic_dec(&kfd_locked);
	846	+
	847	+ atomic_set(&kfd->sram_ecc_flag, 0);
	848	+
	849	+ kfd_smi_event_update_gpu_reset(kfd, true);
	850	+
592	851	return 0;
593	852	}
594	853
..	..	@@ -597,21 +856,23 @@
597	856	return (atomic_read(&kfd_locked) > 0);
598	857	}
599	858
600		-void kgd2kfd_suspend(struct kfd_dev *kfd)
	859	+void kgd2kfd_suspend(struct kfd_dev *kfd, bool run_pm)
601	860	{
602	861	if (!kfd->init_complete)
603	862	return;
604	863
605		- /* For first KFD device suspend all the KFD processes */
606		- if (atomic_inc_return(&kfd_locked) == 1)
607		- kfd_suspend_all_processes();
	864	+ /* for runtime suspend, skip locking kfd */
	865	+ if (!run_pm) {
	866	+ /* For first KFD device suspend all the KFD processes */
	867	+ if (atomic_inc_return(&kfd_locked) == 1)
	868	+ kfd_suspend_all_processes();
	869	+ }
608	870
609	871	kfd->dqm->ops.stop(kfd->dqm);
610		-
611	872	kfd_iommu_suspend(kfd);
612	873	}
613	874
614		-int kgd2kfd_resume(struct kfd_dev *kfd)
	875	+int kgd2kfd_resume(struct kfd_dev *kfd, bool run_pm)
615	876	{
616	877	int ret, count;
617	878
..	..	@@ -622,25 +883,32 @@
622	883	if (ret)
623	884	return ret;
624	885
625		- count = atomic_dec_return(&kfd_locked);
626		- WARN_ONCE(count < 0, "KFD suspend / resume ref. error");
627		- if (count == 0)
628		- ret = kfd_resume_all_processes();
	886	+ /* for runtime resume, skip unlocking kfd */
	887	+ if (!run_pm) {
	888	+ count = atomic_dec_return(&kfd_locked);
	889	+ WARN_ONCE(count < 0, "KFD suspend / resume ref. error");
	890	+ if (count == 0)
	891	+ ret = kfd_resume_all_processes();
	892	+ }
629	893
630	894	return ret;
	895	+}
	896	+
	897	+int kgd2kfd_resume_iommu(struct kfd_dev *kfd)
	898	+{
	899	+ int err = 0;
	900	+
	901	+ err = kfd_iommu_resume(kfd);
	902	+ if (err)
	903	+ dev_err(kfd_device,
	904	+ "Failed to resume IOMMU for device %x:%x\n",
	905	+ kfd->pdev->vendor, kfd->pdev->device);
	906	+ return err;
631	907	}
632	908
633	909	static int kfd_resume(struct kfd_dev *kfd)
634	910	{
635	911	int err = 0;
636		-
637		- err = kfd_iommu_resume(kfd);
638		- if (err) {
639		- dev_err(kfd_device,
640		- "Failed to resume IOMMU for device %x:%x\n",
641		- kfd->pdev->vendor, kfd->pdev->device);
642		- return err;
643		- }
644	912
645	913	err = kfd->dqm->ops.start(kfd->dqm);
646	914	if (err) {
..	..	@@ -655,6 +923,21 @@
655	923	dqm_start_error:
656	924	kfd_iommu_suspend(kfd);
657	925	return err;
	926	+}
	927	+
	928	+static inline void kfd_queue_work(struct workqueue_struct *wq,
	929	+ struct work_struct *work)
	930	+{
	931	+ int cpu, new_cpu;
	932	+
	933	+ cpu = new_cpu = smp_processor_id();
	934	+ do {
	935	+ new_cpu = cpumask_next(new_cpu, cpu_online_mask) % nr_cpu_ids;
	936	+ if (cpu_to_node(new_cpu) == numa_node_id())
	937	+ break;
	938	+ } while (cpu != new_cpu);
	939	+
	940	+ queue_work_on(new_cpu, wq, work);
658	941	}
659	942
660	943	/* This is called directly from KGD at ISR. */
..	..	@@ -679,7 +962,7 @@
679	962	patched_ihre, &is_patched)
680	963	&& enqueue_ih_ring_entry(kfd,
681	964	is_patched ? patched_ihre : ih_ring_entry))
682		- queue_work(kfd->ih_wq, &kfd->interrupt_work);
	965	+ kfd_queue_work(kfd->ih_wq, &kfd->interrupt_work);
683	966
684	967	spin_unlock_irqrestore(&kfd->interrupt_lock, flags);
685	968	}
..	..	@@ -697,6 +980,7 @@
697	980	if (!p)
698	981	return -ESRCH;
699	982
	983	+ WARN(debug_evictions, "Evicting pid %d", p->lead_thread->pid);
700	984	r = kfd_process_evict_queues(p);
701	985
702	986	kfd_unref_process(p);
..	..	@@ -764,6 +1048,8 @@
764	1048	/* During process initialization eviction_work.dwork is initialized
765	1049	* to kfd_evict_bo_worker
766	1050	*/
	1051	+ WARN(debug_evictions, "Scheduling eviction of pid %d in %ld jiffies",
	1052	+ p->lead_thread->pid, delay_jiffies);
767	1053	schedule_delayed_work(&p->eviction_work, delay_jiffies);
768	1054	out:
769	1055	kfd_unref_process(p);
..	..	@@ -952,6 +1238,33 @@
952	1238	return 0;
953	1239	}
954	1240
	1241	+void kgd2kfd_set_sram_ecc_flag(struct kfd_dev *kfd)
	1242	+{
	1243	+ if (kfd)
	1244	+ atomic_inc(&kfd->sram_ecc_flag);
	1245	+}
	1246	+
	1247	+void kfd_inc_compute_active(struct kfd_dev *kfd)
	1248	+{
	1249	+ if (atomic_inc_return(&kfd->compute_profile) == 1)
	1250	+ amdgpu_amdkfd_set_compute_idle(kfd->kgd, false);
	1251	+}
	1252	+
	1253	+void kfd_dec_compute_active(struct kfd_dev *kfd)
	1254	+{
	1255	+ int count = atomic_dec_return(&kfd->compute_profile);
	1256	+
	1257	+ if (count == 0)
	1258	+ amdgpu_amdkfd_set_compute_idle(kfd->kgd, true);
	1259	+ WARN_ONCE(count < 0, "Compute profile ref. count error");
	1260	+}
	1261	+
	1262	+void kgd2kfd_smi_event_throttle(struct kfd_dev *kfd, uint32_t throttle_bitmask)
	1263	+{
	1264	+ if (kfd)
	1265	+ kfd_smi_event_update_thermal_throttling(kfd, throttle_bitmask);
	1266	+}
	1267	+
955	1268	#if defined(CONFIG_DEBUG_FS)
956	1269
957	1270	/* This function will send a package to HIQ to hang the HWS