~hc/RK356X_SDK_RELEASE.git

..	..	@@ -24,9 +24,17 @@
24	24	*/
25	25
26	26	#include <linux/kthread.h>
27		-#include <drm/drmP.h>
28		-#include <linux/debugfs.h>
	27	+#include <linux/pci.h>
	28	+#include <linux/uaccess.h>
	29	+#include <linux/pm_runtime.h>
	30	+#include <linux/poll.h>
	31	+#include <drm/drm_debugfs.h>
	32	+
29	33	#include "amdgpu.h"
	34	+#include "amdgpu_pm.h"
	35	+#include "amdgpu_dm_debugfs.h"
	36	+#include "amdgpu_ras.h"
	37	+#include "amdgpu_rap.h"
30	38
31	39	/**
32	40	* amdgpu_debugfs_add_files - Add simple debugfs entries
..	..	@@ -61,13 +69,91 @@
61	69	adev->debugfs_count = i;
62	70	#if defined(CONFIG_DEBUG_FS)
63	71	drm_debugfs_create_files(files, nfiles,
64		- adev->ddev->primary->debugfs_root,
65		- adev->ddev->primary);
	72	+ adev_to_drm(adev)->primary->debugfs_root,
	73	+ adev_to_drm(adev)->primary);
	74	+#endif
	75	+ return 0;
	76	+}
	77	+
	78	+int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev)
	79	+{
	80	+#if defined(CONFIG_DEBUG_FS)
	81	+ unsigned long timeout = 600 * HZ;
	82	+ int ret;
	83	+
	84	+ wake_up_interruptible(&adev->autodump.gpu_hang);
	85	+
	86	+ ret = wait_for_completion_interruptible_timeout(&adev->autodump.dumping, timeout);
	87	+ if (ret == 0) {
	88	+ pr_err("autodump: timeout, move on to gpu recovery\n");
	89	+ return -ETIMEDOUT;
	90	+ }
66	91	#endif
67	92	return 0;
68	93	}
69	94
70	95	#if defined(CONFIG_DEBUG_FS)
	96	+
	97	+static int amdgpu_debugfs_autodump_open(struct inode inode, struct file file)
	98	+{
	99	+ struct amdgpu_device *adev = inode->i_private;
	100	+ int ret;
	101	+
	102	+ file->private_data = adev;
	103	+
	104	+ ret = down_read_killable(&adev->reset_sem);
	105	+ if (ret)
	106	+ return ret;
	107	+
	108	+ if (adev->autodump.dumping.done) {
	109	+ reinit_completion(&adev->autodump.dumping);
	110	+ ret = 0;
	111	+ } else {
	112	+ ret = -EBUSY;
	113	+ }
	114	+
	115	+ up_read(&adev->reset_sem);
	116	+
	117	+ return ret;
	118	+}
	119	+
	120	+static int amdgpu_debugfs_autodump_release(struct inode inode, struct file file)
	121	+{
	122	+ struct amdgpu_device *adev = file->private_data;
	123	+
	124	+ complete_all(&adev->autodump.dumping);
	125	+ return 0;
	126	+}
	127	+
	128	+static unsigned int amdgpu_debugfs_autodump_poll(struct file file, struct poll_table_struct poll_table)
	129	+{
	130	+ struct amdgpu_device *adev = file->private_data;
	131	+
	132	+ poll_wait(file, &adev->autodump.gpu_hang, poll_table);
	133	+
	134	+ if (amdgpu_in_reset(adev))
	135	+ return POLLIN \| POLLRDNORM \| POLLWRNORM;
	136	+
	137	+ return 0;
	138	+}
	139	+
	140	+static const struct file_operations autodump_debug_fops = {
	141	+ .owner = THIS_MODULE,
	142	+ .open = amdgpu_debugfs_autodump_open,
	143	+ .poll = amdgpu_debugfs_autodump_poll,
	144	+ .release = amdgpu_debugfs_autodump_release,
	145	+};
	146	+
	147	+static void amdgpu_debugfs_autodump_init(struct amdgpu_device *adev)
	148	+{
	149	+ init_completion(&adev->autodump.dumping);
	150	+ complete_all(&adev->autodump.dumping);
	151	+ init_waitqueue_head(&adev->autodump.gpu_hang);
	152	+
	153	+ debugfs_create_file("amdgpu_autodump", 0600,
	154	+ adev_to_drm(adev)->primary->debugfs_root,
	155	+ adev, &autodump_debug_fops);
	156	+}
71	157
72	158	/**
73	159	* amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
..	..	@@ -103,10 +189,10 @@
103	189	ssize_t result = 0;
104	190	int r;
105	191	bool pm_pg_lock, use_bank, use_ring;
106		- unsigned instance_bank, sh_bank, se_bank, me, pipe, queue;
	192	+ unsigned instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
107	193
108	194	pm_pg_lock = use_bank = use_ring = false;
109		- instance_bank = sh_bank = se_bank = me = pipe = queue = 0;
	195	+ instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
110	196
111	197	if (size & 0x3 \|\| *pos & 0x3 \|\|
112	198	((pos & (1ULL << 62)) && (pos & (1ULL << 61))))
..	..	@@ -126,30 +212,47 @@
126	212	sh_bank = 0xFFFFFFFF;
127	213	if (instance_bank == 0x3FF)
128	214	instance_bank = 0xFFFFFFFF;
129		- use_bank = 1;
	215	+ use_bank = true;
130	216	} else if (*pos & (1ULL << 61)) {
131	217
132	218	me = (*pos & GENMASK_ULL(33, 24)) >> 24;
133	219	pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
134	220	queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
	221	+ vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
135	222
136		- use_ring = 1;
	223	+ use_ring = true;
137	224	} else {
138		- use_bank = use_ring = 0;
	225	+ use_bank = use_ring = false;
139	226	}
140	227
141	228	*pos &= (1UL << 22) - 1;
142	229
	230	+ r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
	231	+ if (r < 0) {
	232	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	233	+ return r;
	234	+ }
	235	+
	236	+ r = amdgpu_virt_enable_access_debugfs(adev);
	237	+ if (r < 0) {
	238	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	239	+ return r;
	240	+ }
	241	+
143	242	if (use_bank) {
144	243	if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) \|\|
145		- (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines))
	244	+ (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
	245	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	246	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	247	+ amdgpu_virt_disable_access_debugfs(adev);
146	248	return -EINVAL;
	249	+ }
147	250	mutex_lock(&adev->grbm_idx_mutex);
148	251	amdgpu_gfx_select_se_sh(adev, se_bank,
149	252	sh_bank, instance_bank);
150	253	} else if (use_ring) {
151	254	mutex_lock(&adev->srbm_mutex);
152		- amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue);
	255	+ amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid);
153	256	}
154	257
155	258	if (pm_pg_lock)
..	..	@@ -158,16 +261,13 @@
158	261	while (size) {
159	262	uint32_t value;
160	263
161		- if (*pos > adev->rmmio_size)
162		- goto end;
163		-
164	264	if (read) {
165	265	value = RREG32(*pos >> 2);
166	266	r = put_user(value, (uint32_t *)buf);
167	267	} else {
168	268	r = get_user(value, (uint32_t *)buf);
169	269	if (!r)
170		- WREG32(*pos >> 2, value);
	270	+ amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value);
171	271	}
172	272	if (r) {
173	273	result = r;
..	..	@@ -185,13 +285,17 @@
185	285	amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
186	286	mutex_unlock(&adev->grbm_idx_mutex);
187	287	} else if (use_ring) {
188		- amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0);
	288	+ amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
189	289	mutex_unlock(&adev->srbm_mutex);
190	290	}
191	291
192	292	if (pm_pg_lock)
193	293	mutex_unlock(&adev->pm.mutex);
194	294
	295	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	296	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	297	+
	298	+ amdgpu_virt_disable_access_debugfs(adev);
195	299	return result;
196	300	}
197	301
..	..	@@ -236,13 +340,29 @@
236	340	if (size & 0x3 \|\| *pos & 0x3)
237	341	return -EINVAL;
238	342
	343	+ r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
	344	+ if (r < 0) {
	345	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	346	+ return r;
	347	+ }
	348	+
	349	+ r = amdgpu_virt_enable_access_debugfs(adev);
	350	+ if (r < 0) {
	351	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	352	+ return r;
	353	+ }
	354	+
239	355	while (size) {
240	356	uint32_t value;
241	357
242	358	value = RREG32_PCIE(*pos);
243	359	r = put_user(value, (uint32_t *)buf);
244		- if (r)
	360	+ if (r) {
	361	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	362	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	363	+ amdgpu_virt_disable_access_debugfs(adev);
245	364	return r;
	365	+ }
246	366
247	367	result += 4;
248	368	buf += 4;
..	..	@@ -250,6 +370,10 @@
250	370	size -= 4;
251	371	}
252	372
	373	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	374	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	375	+
	376	+ amdgpu_virt_disable_access_debugfs(adev);
253	377	return result;
254	378	}
255	379
..	..	@@ -275,12 +399,28 @@
275	399	if (size & 0x3 \|\| *pos & 0x3)
276	400	return -EINVAL;
277	401
	402	+ r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
	403	+ if (r < 0) {
	404	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	405	+ return r;
	406	+ }
	407	+
	408	+ r = amdgpu_virt_enable_access_debugfs(adev);
	409	+ if (r < 0) {
	410	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	411	+ return r;
	412	+ }
	413	+
278	414	while (size) {
279	415	uint32_t value;
280	416
281	417	r = get_user(value, (uint32_t *)buf);
282		- if (r)
	418	+ if (r) {
	419	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	420	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	421	+ amdgpu_virt_disable_access_debugfs(adev);
283	422	return r;
	423	+ }
284	424
285	425	WREG32_PCIE(*pos, value);
286	426
..	..	@@ -290,6 +430,10 @@
290	430	size -= 4;
291	431	}
292	432
	433	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	434	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	435	+
	436	+ amdgpu_virt_disable_access_debugfs(adev);
293	437	return result;
294	438	}
295	439
..	..	@@ -315,13 +459,29 @@
315	459	if (size & 0x3 \|\| *pos & 0x3)
316	460	return -EINVAL;
317	461
	462	+ r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
	463	+ if (r < 0) {
	464	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	465	+ return r;
	466	+ }
	467	+
	468	+ r = amdgpu_virt_enable_access_debugfs(adev);
	469	+ if (r < 0) {
	470	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	471	+ return r;
	472	+ }
	473	+
318	474	while (size) {
319	475	uint32_t value;
320	476
321	477	value = RREG32_DIDT(*pos >> 2);
322	478	r = put_user(value, (uint32_t *)buf);
323		- if (r)
	479	+ if (r) {
	480	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	481	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	482	+ amdgpu_virt_disable_access_debugfs(adev);
324	483	return r;
	484	+ }
325	485
326	486	result += 4;
327	487	buf += 4;
..	..	@@ -329,6 +489,10 @@
329	489	size -= 4;
330	490	}
331	491
	492	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	493	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	494	+
	495	+ amdgpu_virt_disable_access_debugfs(adev);
332	496	return result;
333	497	}
334	498
..	..	@@ -354,12 +518,28 @@
354	518	if (size & 0x3 \|\| *pos & 0x3)
355	519	return -EINVAL;
356	520
	521	+ r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
	522	+ if (r < 0) {
	523	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	524	+ return r;
	525	+ }
	526	+
	527	+ r = amdgpu_virt_enable_access_debugfs(adev);
	528	+ if (r < 0) {
	529	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	530	+ return r;
	531	+ }
	532	+
357	533	while (size) {
358	534	uint32_t value;
359	535
360	536	r = get_user(value, (uint32_t *)buf);
361		- if (r)
	537	+ if (r) {
	538	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	539	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	540	+ amdgpu_virt_disable_access_debugfs(adev);
362	541	return r;
	542	+ }
363	543
364	544	WREG32_DIDT(*pos >> 2, value);
365	545
..	..	@@ -369,6 +549,10 @@
369	549	size -= 4;
370	550	}
371	551
	552	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	553	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	554	+
	555	+ amdgpu_virt_disable_access_debugfs(adev);
372	556	return result;
373	557	}
374	558
..	..	@@ -394,13 +578,29 @@
394	578	if (size & 0x3 \|\| *pos & 0x3)
395	579	return -EINVAL;
396	580
	581	+ r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
	582	+ if (r < 0) {
	583	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	584	+ return r;
	585	+ }
	586	+
	587	+ r = amdgpu_virt_enable_access_debugfs(adev);
	588	+ if (r < 0) {
	589	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	590	+ return r;
	591	+ }
	592	+
397	593	while (size) {
398	594	uint32_t value;
399	595
400	596	value = RREG32_SMC(*pos);
401	597	r = put_user(value, (uint32_t *)buf);
402		- if (r)
	598	+ if (r) {
	599	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	600	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	601	+ amdgpu_virt_disable_access_debugfs(adev);
403	602	return r;
	603	+ }
404	604
405	605	result += 4;
406	606	buf += 4;
..	..	@@ -408,6 +608,10 @@
408	608	size -= 4;
409	609	}
410	610
	611	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	612	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	613	+
	614	+ amdgpu_virt_disable_access_debugfs(adev);
411	615	return result;
412	616	}
413	617
..	..	@@ -433,12 +637,28 @@
433	637	if (size & 0x3 \|\| *pos & 0x3)
434	638	return -EINVAL;
435	639
	640	+ r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
	641	+ if (r < 0) {
	642	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	643	+ return r;
	644	+ }
	645	+
	646	+ r = amdgpu_virt_enable_access_debugfs(adev);
	647	+ if (r < 0) {
	648	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	649	+ return r;
	650	+ }
	651	+
436	652	while (size) {
437	653	uint32_t value;
438	654
439	655	r = get_user(value, (uint32_t *)buf);
440		- if (r)
	656	+ if (r) {
	657	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	658	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	659	+ amdgpu_virt_disable_access_debugfs(adev);
441	660	return r;
	661	+ }
442	662
443	663	WREG32_SMC(*pos, value);
444	664
..	..	@@ -448,6 +668,10 @@
448	668	size -= 4;
449	669	}
450	670
	671	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	672	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	673	+
	674	+ amdgpu_virt_disable_access_debugfs(adev);
451	675	return result;
452	676	}
453	677
..	..	@@ -571,13 +795,33 @@
571	795	idx = *pos >> 2;
572	796
573	797	valuesize = sizeof(values);
574		- if (adev->powerplay.pp_funcs && adev->powerplay.pp_funcs->read_sensor)
575		- r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
576		- else
577		- return -EINVAL;
578	798
579		- if (size > valuesize)
	799	+ r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
	800	+ if (r < 0) {
	801	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	802	+ return r;
	803	+ }
	804	+
	805	+ r = amdgpu_virt_enable_access_debugfs(adev);
	806	+ if (r < 0) {
	807	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	808	+ return r;
	809	+ }
	810	+
	811	+ r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
	812	+
	813	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	814	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	815	+
	816	+ if (r) {
	817	+ amdgpu_virt_disable_access_debugfs(adev);
	818	+ return r;
	819	+ }
	820	+
	821	+ if (size > valuesize) {
	822	+ amdgpu_virt_disable_access_debugfs(adev);
580	823	return -EINVAL;
	824	+ }
581	825
582	826	outsize = 0;
583	827	x = 0;
..	..	@@ -590,6 +834,7 @@
590	834	}
591	835	}
592	836
	837	+ amdgpu_virt_disable_access_debugfs(adev);
593	838	return !r ? outsize : r;
594	839	}
595	840
..	..	@@ -633,6 +878,18 @@
633	878	wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
634	879	simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
635	880
	881	+ r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
	882	+ if (r < 0) {
	883	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	884	+ return r;
	885	+ }
	886	+
	887	+ r = amdgpu_virt_enable_access_debugfs(adev);
	888	+ if (r < 0) {
	889	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	890	+ return r;
	891	+ }
	892	+
636	893	/* switch to the specific se/sh/cu */
637	894	mutex_lock(&adev->grbm_idx_mutex);
638	895	amdgpu_gfx_select_se_sh(adev, se, sh, cu);
..	..	@@ -644,16 +901,23 @@
644	901	amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
645	902	mutex_unlock(&adev->grbm_idx_mutex);
646	903
647		- if (!x)
	904	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	905	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	906	+
	907	+ if (!x) {
	908	+ amdgpu_virt_disable_access_debugfs(adev);
648	909	return -EINVAL;
	910	+ }
649	911
650	912	while (size && (offset < x * 4)) {
651	913	uint32_t value;
652	914
653	915	value = data[offset >> 2];
654	916	r = put_user(value, (uint32_t *)buf);
655		- if (r)
	917	+ if (r) {
	918	+ amdgpu_virt_disable_access_debugfs(adev);
656	919	return r;
	920	+ }
657	921
658	922	result += 4;
659	923	buf += 4;
..	..	@@ -661,6 +925,7 @@
661	925	size -= 4;
662	926	}
663	927
	928	+ amdgpu_virt_disable_access_debugfs(adev);
664	929	return result;
665	930	}
666	931
..	..	@@ -711,6 +976,14 @@
711	976	if (!data)
712	977	return -ENOMEM;
713	978
	979	+ r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
	980	+ if (r < 0)
	981	+ goto err;
	982	+
	983	+ r = amdgpu_virt_enable_access_debugfs(adev);
	984	+ if (r < 0)
	985	+ goto err;
	986	+
714	987	/* switch to the specific se/sh/cu */
715	988	mutex_lock(&adev->grbm_idx_mutex);
716	989	amdgpu_gfx_select_se_sh(adev, se, sh, cu);
..	..	@@ -726,13 +999,16 @@
726	999	amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
727	1000	mutex_unlock(&adev->grbm_idx_mutex);
728	1001
	1002	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	1003	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	1004	+
729	1005	while (size) {
730	1006	uint32_t value;
731	1007
732	1008	value = data[result >> 2];
733	1009	r = put_user(value, (uint32_t *)buf);
734	1010	if (r) {
735		- result = r;
	1011	+ amdgpu_virt_disable_access_debugfs(adev);
736	1012	goto err;
737	1013	}
738	1014
..	..	@@ -741,8 +1017,115 @@
741	1017	size -= 4;
742	1018	}
743	1019
744		-err:
745	1020	kfree(data);
	1021	+ amdgpu_virt_disable_access_debugfs(adev);
	1022	+ return result;
	1023	+
	1024	+err:
	1025	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	1026	+ kfree(data);
	1027	+ return r;
	1028	+}
	1029	+
	1030	+/**
	1031	+ * amdgpu_debugfs_regs_gfxoff_write - Enable/disable GFXOFF
	1032	+ *
	1033	+ * @f: open file handle
	1034	+ * @buf: User buffer to write data from
	1035	+ * @size: Number of bytes to write
	1036	+ * @pos: Offset to seek to
	1037	+ *
	1038	+ * Write a 32-bit zero to disable or a 32-bit non-zero to enable
	1039	+ */
	1040	+static ssize_t amdgpu_debugfs_gfxoff_write(struct file f, const char __user buf,
	1041	+ size_t size, loff_t *pos)
	1042	+{
	1043	+ struct amdgpu_device *adev = file_inode(f)->i_private;
	1044	+ ssize_t result = 0;
	1045	+ int r;
	1046	+
	1047	+ if (size & 0x3 \|\| *pos & 0x3)
	1048	+ return -EINVAL;
	1049	+
	1050	+ r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
	1051	+ if (r < 0) {
	1052	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	1053	+ return r;
	1054	+ }
	1055	+
	1056	+ while (size) {
	1057	+ uint32_t value;
	1058	+
	1059	+ r = get_user(value, (uint32_t *)buf);
	1060	+ if (r) {
	1061	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	1062	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	1063	+ return r;
	1064	+ }
	1065	+
	1066	+ amdgpu_gfx_off_ctrl(adev, value ? true : false);
	1067	+
	1068	+ result += 4;
	1069	+ buf += 4;
	1070	+ *pos += 4;
	1071	+ size -= 4;
	1072	+ }
	1073	+
	1074	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	1075	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	1076	+
	1077	+ return result;
	1078	+}
	1079	+
	1080	+
	1081	+/**
	1082	+ * amdgpu_debugfs_regs_gfxoff_status - read gfxoff status
	1083	+ *
	1084	+ * @f: open file handle
	1085	+ * @buf: User buffer to store read data in
	1086	+ * @size: Number of bytes to read
	1087	+ * @pos: Offset to seek to
	1088	+ */
	1089	+static ssize_t amdgpu_debugfs_gfxoff_read(struct file f, char __user buf,
	1090	+ size_t size, loff_t *pos)
	1091	+{
	1092	+ struct amdgpu_device *adev = file_inode(f)->i_private;
	1093	+ ssize_t result = 0;
	1094	+ int r;
	1095	+
	1096	+ if (size & 0x3 \|\| *pos & 0x3)
	1097	+ return -EINVAL;
	1098	+
	1099	+ r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
	1100	+ if (r < 0)
	1101	+ return r;
	1102	+
	1103	+ while (size) {
	1104	+ uint32_t value;
	1105	+
	1106	+ r = amdgpu_get_gfx_off_status(adev, &value);
	1107	+ if (r) {
	1108	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	1109	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	1110	+ return r;
	1111	+ }
	1112	+
	1113	+ r = put_user(value, (uint32_t *)buf);
	1114	+ if (r) {
	1115	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	1116	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	1117	+ return r;
	1118	+ }
	1119	+
	1120	+ result += 4;
	1121	+ buf += 4;
	1122	+ *pos += 4;
	1123	+ size -= 4;
	1124	+ }
	1125	+
	1126	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	1127	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	1128	+
746	1129	return result;
747	1130	}
748	1131
..	..	@@ -794,6 +1177,13 @@
794	1177	.llseek = default_llseek
795	1178	};
796	1179
	1180	+static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
	1181	+ .owner = THIS_MODULE,
	1182	+ .read = amdgpu_debugfs_gfxoff_read,
	1183	+ .write = amdgpu_debugfs_gfxoff_write,
	1184	+ .llseek = default_llseek
	1185	+};
	1186	+
797	1187	static const struct file_operations *debugfs_regs[] = {
798	1188	&amdgpu_debugfs_regs_fops,
799	1189	&amdgpu_debugfs_regs_didt_fops,
..	..	@@ -803,6 +1193,7 @@
803	1193	&amdgpu_debugfs_sensors_fops,
804	1194	&amdgpu_debugfs_wave_fops,
805	1195	&amdgpu_debugfs_gpr_fops,
	1196	+ &amdgpu_debugfs_gfxoff_fops,
806	1197	};
807	1198
808	1199	static const char *debugfs_regs_names[] = {
..	..	@@ -814,6 +1205,7 @@
814	1205	"amdgpu_sensors",
815	1206	"amdgpu_wave",
816	1207	"amdgpu_gpr",
	1208	+ "amdgpu_gfxoff",
817	1209	};
818	1210
819	1211	/**
..	..	@@ -824,23 +1216,15 @@
824	1216	*/
825	1217	int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
826	1218	{
827		- struct drm_minor *minor = adev->ddev->primary;
	1219	+ struct drm_minor *minor = adev_to_drm(adev)->primary;
828	1220	struct dentry ent, root = minor->debugfs_root;
829		- unsigned i, j;
	1221	+ unsigned int i;
830	1222
831	1223	for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
832	1224	ent = debugfs_create_file(debugfs_regs_names[i],
833	1225	S_IFREG \| S_IRUGO, root,
834	1226	adev, debugfs_regs[i]);
835		- if (IS_ERR(ent)) {
836		- for (j = 0; j < i; j++) {
837		- debugfs_remove(adev->debugfs_regs[i]);
838		- adev->debugfs_regs[i] = NULL;
839		- }
840		- return PTR_ERR(ent);
841		- }
842		-
843		- if (!i)
	1227	+ if (!i && !IS_ERR_OR_NULL(ent))
844	1228	i_size_write(ent->d_inode, adev->rmmio_size);
845	1229	adev->debugfs_regs[i] = ent;
846	1230	}
..	..	@@ -848,24 +1232,23 @@
848	1232	return 0;
849	1233	}
850	1234
851		-void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev)
852		-{
853		- unsigned i;
854		-
855		- for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
856		- if (adev->debugfs_regs[i]) {
857		- debugfs_remove(adev->debugfs_regs[i]);
858		- adev->debugfs_regs[i] = NULL;
859		- }
860		- }
861		-}
862		-
863	1235	static int amdgpu_debugfs_test_ib(struct seq_file m, void data)
864	1236	{
865	1237	struct drm_info_node node = (struct drm_info_node ) m->private;
866	1238	struct drm_device *dev = node->minor->dev;
867		- struct amdgpu_device *adev = dev->dev_private;
	1239	+ struct amdgpu_device *adev = drm_to_adev(dev);
868	1240	int r = 0, i;
	1241	+
	1242	+ r = pm_runtime_get_sync(dev->dev);
	1243	+ if (r < 0) {
	1244	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	1245	+ return r;
	1246	+ }
	1247	+
	1248	+ /* Avoid accidently unparking the sched thread during GPU reset */
	1249	+ r = down_read_killable(&adev->reset_sem);
	1250	+ if (r)
	1251	+ return r;
869	1252
870	1253	/* hold on the scheduler */
871	1254	for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
..	..	@@ -892,6 +1275,11 @@
892	1275	kthread_unpark(ring->sched.thread);
893	1276	}
894	1277
	1278	+ up_read(&adev->reset_sem);
	1279	+
	1280	+ pm_runtime_mark_last_busy(dev->dev);
	1281	+ pm_runtime_put_autosuspend(dev->dev);
	1282	+
895	1283	return 0;
896	1284	}
897	1285
..	..	@@ -899,7 +1287,7 @@
899	1287	{
900	1288	struct drm_info_node node = (struct drm_info_node ) m->private;
901	1289	struct drm_device *dev = node->minor->dev;
902		- struct amdgpu_device *adev = dev->dev_private;
	1290	+ struct amdgpu_device *adev = drm_to_adev(dev);
903	1291
904	1292	seq_write(m, adev->bios, adev->bios_size);
905	1293	return 0;
..	..	@@ -909,9 +1297,20 @@
909	1297	{
910	1298	struct drm_info_node node = (struct drm_info_node )m->private;
911	1299	struct drm_device *dev = node->minor->dev;
912		- struct amdgpu_device *adev = dev->dev_private;
	1300	+ struct amdgpu_device *adev = drm_to_adev(dev);
	1301	+ int r;
	1302	+
	1303	+ r = pm_runtime_get_sync(dev->dev);
	1304	+ if (r < 0) {
	1305	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	1306	+ return r;
	1307	+ }
913	1308
914	1309	seq_printf(m, "(%d)\n", amdgpu_bo_evict_vram(adev));
	1310	+
	1311	+ pm_runtime_mark_last_busy(dev->dev);
	1312	+ pm_runtime_put_autosuspend(dev->dev);
	1313	+
915	1314	return 0;
916	1315	}
917	1316
..	..	@@ -919,9 +1318,20 @@
919	1318	{
920	1319	struct drm_info_node node = (struct drm_info_node )m->private;
921	1320	struct drm_device *dev = node->minor->dev;
922		- struct amdgpu_device *adev = dev->dev_private;
	1321	+ struct amdgpu_device *adev = drm_to_adev(dev);
	1322	+ int r;
	1323	+
	1324	+ r = pm_runtime_get_sync(dev->dev);
	1325	+ if (r < 0) {
	1326	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	1327	+ return r;
	1328	+ }
923	1329
924	1330	seq_printf(m, "(%d)\n", ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_TT));
	1331	+
	1332	+ pm_runtime_mark_last_busy(dev->dev);
	1333	+ pm_runtime_put_autosuspend(dev->dev);
	1334	+
925	1335	return 0;
926	1336	}
927	1337
..	..	@@ -932,8 +1342,299 @@
932	1342	{"amdgpu_evict_gtt", &amdgpu_debugfs_evict_gtt},
933	1343	};
934	1344
	1345	+static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
	1346	+ struct dma_fence **fences)
	1347	+{
	1348	+ struct amdgpu_fence_driver *drv = &ring->fence_drv;
	1349	+ uint32_t sync_seq, last_seq;
	1350	+
	1351	+ last_seq = atomic_read(&ring->fence_drv.last_seq);
	1352	+ sync_seq = ring->fence_drv.sync_seq;
	1353	+
	1354	+ last_seq &= drv->num_fences_mask;
	1355	+ sync_seq &= drv->num_fences_mask;
	1356	+
	1357	+ do {
	1358	+ struct dma_fence fence, *ptr;
	1359	+
	1360	+ ++last_seq;
	1361	+ last_seq &= drv->num_fences_mask;
	1362	+ ptr = &drv->fences[last_seq];
	1363	+
	1364	+ fence = rcu_dereference_protected(*ptr, 1);
	1365	+ RCU_INIT_POINTER(*ptr, NULL);
	1366	+
	1367	+ if (!fence)
	1368	+ continue;
	1369	+
	1370	+ fences[last_seq] = fence;
	1371	+
	1372	+ } while (last_seq != sync_seq);
	1373	+}
	1374	+
	1375	+static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
	1376	+ int length)
	1377	+{
	1378	+ int i;
	1379	+ struct dma_fence *fence;
	1380	+
	1381	+ for (i = 0; i < length; i++) {
	1382	+ fence = fences[i];
	1383	+ if (!fence)
	1384	+ continue;
	1385	+ dma_fence_signal(fence);
	1386	+ dma_fence_put(fence);
	1387	+ }
	1388	+}
	1389	+
	1390	+static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
	1391	+{
	1392	+ struct drm_sched_job *s_job;
	1393	+ struct dma_fence *fence;
	1394	+
	1395	+ spin_lock(&sched->job_list_lock);
	1396	+ list_for_each_entry(s_job, &sched->ring_mirror_list, node) {
	1397	+ fence = sched->ops->run_job(s_job);
	1398	+ dma_fence_put(fence);
	1399	+ }
	1400	+ spin_unlock(&sched->job_list_lock);
	1401	+}
	1402	+
	1403	+static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
	1404	+{
	1405	+ struct amdgpu_job *job;
	1406	+ struct drm_sched_job s_job, tmp;
	1407	+ uint32_t preempt_seq;
	1408	+ struct dma_fence fence, *ptr;
	1409	+ struct amdgpu_fence_driver *drv = &ring->fence_drv;
	1410	+ struct drm_gpu_scheduler *sched = &ring->sched;
	1411	+ bool preempted = true;
	1412	+
	1413	+ if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
	1414	+ return;
	1415	+
	1416	+ preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
	1417	+ if (preempt_seq <= atomic_read(&drv->last_seq)) {
	1418	+ preempted = false;
	1419	+ goto no_preempt;
	1420	+ }
	1421	+
	1422	+ preempt_seq &= drv->num_fences_mask;
	1423	+ ptr = &drv->fences[preempt_seq];
	1424	+ fence = rcu_dereference_protected(*ptr, 1);
	1425	+
	1426	+no_preempt:
	1427	+ spin_lock(&sched->job_list_lock);
	1428	+ list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
	1429	+ if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
	1430	+ /* remove job from ring_mirror_list */
	1431	+ list_del_init(&s_job->node);
	1432	+ sched->ops->free_job(s_job);
	1433	+ continue;
	1434	+ }
	1435	+ job = to_amdgpu_job(s_job);
	1436	+ if (preempted && job->fence == fence)
	1437	+ /* mark the job as preempted */
	1438	+ job->preemption_status \|= AMDGPU_IB_PREEMPTED;
	1439	+ }
	1440	+ spin_unlock(&sched->job_list_lock);
	1441	+}
	1442	+
	1443	+static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
	1444	+{
	1445	+ int r, resched, length;
	1446	+ struct amdgpu_ring *ring;
	1447	+ struct dma_fence **fences = NULL;
	1448	+ struct amdgpu_device adev = (struct amdgpu_device )data;
	1449	+
	1450	+ if (val >= AMDGPU_MAX_RINGS)
	1451	+ return -EINVAL;
	1452	+
	1453	+ ring = adev->rings[val];
	1454	+
	1455	+ if (!ring \|\| !ring->funcs->preempt_ib \|\| !ring->sched.thread)
	1456	+ return -EINVAL;
	1457	+
	1458	+ /* the last preemption failed */
	1459	+ if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
	1460	+ return -EBUSY;
	1461	+
	1462	+ length = ring->fence_drv.num_fences_mask + 1;
	1463	+ fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
	1464	+ if (!fences)
	1465	+ return -ENOMEM;
	1466	+
	1467	+ /* Avoid accidently unparking the sched thread during GPU reset */
	1468	+ r = down_read_killable(&adev->reset_sem);
	1469	+ if (r)
	1470	+ goto pro_end;
	1471	+
	1472	+ /* stop the scheduler */
	1473	+ kthread_park(ring->sched.thread);
	1474	+
	1475	+ resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
	1476	+
	1477	+ /* preempt the IB */
	1478	+ r = amdgpu_ring_preempt_ib(ring);
	1479	+ if (r) {
	1480	+ DRM_WARN("failed to preempt ring %d\n", ring->idx);
	1481	+ goto failure;
	1482	+ }
	1483	+
	1484	+ amdgpu_fence_process(ring);
	1485	+
	1486	+ if (atomic_read(&ring->fence_drv.last_seq) !=
	1487	+ ring->fence_drv.sync_seq) {
	1488	+ DRM_INFO("ring %d was preempted\n", ring->idx);
	1489	+
	1490	+ amdgpu_ib_preempt_mark_partial_job(ring);
	1491	+
	1492	+ /* swap out the old fences */
	1493	+ amdgpu_ib_preempt_fences_swap(ring, fences);
	1494	+
	1495	+ amdgpu_fence_driver_force_completion(ring);
	1496	+
	1497	+ /* resubmit unfinished jobs */
	1498	+ amdgpu_ib_preempt_job_recovery(&ring->sched);
	1499	+
	1500	+ /* wait for jobs finished */
	1501	+ amdgpu_fence_wait_empty(ring);
	1502	+
	1503	+ /* signal the old fences */
	1504	+ amdgpu_ib_preempt_signal_fences(fences, length);
	1505	+ }
	1506	+
	1507	+failure:
	1508	+ /* restart the scheduler */
	1509	+ kthread_unpark(ring->sched.thread);
	1510	+
	1511	+ up_read(&adev->reset_sem);
	1512	+
	1513	+ ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
	1514	+
	1515	+pro_end:
	1516	+ kfree(fences);
	1517	+
	1518	+ return r;
	1519	+}
	1520	+
	1521	+static int amdgpu_debugfs_sclk_set(void *data, u64 val)
	1522	+{
	1523	+ int ret = 0;
	1524	+ uint32_t max_freq, min_freq;
	1525	+ struct amdgpu_device adev = (struct amdgpu_device )data;
	1526	+
	1527	+ if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
	1528	+ return -EINVAL;
	1529	+
	1530	+ ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
	1531	+ if (ret < 0) {
	1532	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	1533	+ return ret;
	1534	+ }
	1535	+
	1536	+ if (is_support_sw_smu(adev)) {
	1537	+ ret = smu_get_dpm_freq_range(&adev->smu, SMU_SCLK, &min_freq, &max_freq);
	1538	+ if (ret \|\| val > max_freq \|\| val < min_freq)
	1539	+ return -EINVAL;
	1540	+ ret = smu_set_soft_freq_range(&adev->smu, SMU_SCLK, (uint32_t)val, (uint32_t)val);
	1541	+ } else {
	1542	+ return 0;
	1543	+ }
	1544	+
	1545	+ pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
	1546	+ pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
	1547	+
	1548	+ if (ret)
	1549	+ return -EINVAL;
	1550	+
	1551	+ return 0;
	1552	+}
	1553	+
	1554	+DEFINE_SIMPLE_ATTRIBUTE(fops_ib_preempt, NULL,
	1555	+ amdgpu_debugfs_ib_preempt, "%llu\n");
	1556	+
	1557	+DEFINE_SIMPLE_ATTRIBUTE(fops_sclk_set, NULL,
	1558	+ amdgpu_debugfs_sclk_set, "%llu\n");
	1559	+
935	1560	int amdgpu_debugfs_init(struct amdgpu_device *adev)
936	1561	{
	1562	+ int r, i;
	1563	+
	1564	+ adev->debugfs_preempt =
	1565	+ debugfs_create_file("amdgpu_preempt_ib", 0600,
	1566	+ adev_to_drm(adev)->primary->debugfs_root, adev,
	1567	+ &fops_ib_preempt);
	1568	+ if (!(adev->debugfs_preempt)) {
	1569	+ DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
	1570	+ return -EIO;
	1571	+ }
	1572	+
	1573	+ adev->smu.debugfs_sclk =
	1574	+ debugfs_create_file("amdgpu_force_sclk", 0200,
	1575	+ adev_to_drm(adev)->primary->debugfs_root, adev,
	1576	+ &fops_sclk_set);
	1577	+ if (!(adev->smu.debugfs_sclk)) {
	1578	+ DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
	1579	+ return -EIO;
	1580	+ }
	1581	+
	1582	+ /* Register debugfs entries for amdgpu_ttm */
	1583	+ r = amdgpu_ttm_debugfs_init(adev);
	1584	+ if (r) {
	1585	+ DRM_ERROR("Failed to init debugfs\n");
	1586	+ return r;
	1587	+ }
	1588	+
	1589	+ r = amdgpu_debugfs_pm_init(adev);
	1590	+ if (r) {
	1591	+ DRM_ERROR("Failed to register debugfs file for dpm!\n");
	1592	+ return r;
	1593	+ }
	1594	+
	1595	+ if (amdgpu_debugfs_sa_init(adev)) {
	1596	+ dev_err(adev->dev, "failed to register debugfs file for SA\n");
	1597	+ }
	1598	+
	1599	+ if (amdgpu_debugfs_fence_init(adev))
	1600	+ dev_err(adev->dev, "fence debugfs file creation failed\n");
	1601	+
	1602	+ r = amdgpu_debugfs_gem_init(adev);
	1603	+ if (r)
	1604	+ DRM_ERROR("registering gem debugfs failed (%d).\n", r);
	1605	+
	1606	+ r = amdgpu_debugfs_regs_init(adev);
	1607	+ if (r)
	1608	+ DRM_ERROR("registering register debugfs failed (%d).\n", r);
	1609	+
	1610	+ r = amdgpu_debugfs_firmware_init(adev);
	1611	+ if (r)
	1612	+ DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
	1613	+
	1614	+#if defined(CONFIG_DRM_AMD_DC)
	1615	+ if (amdgpu_device_has_dc_support(adev)) {
	1616	+ if (dtn_debugfs_init(adev))
	1617	+ DRM_ERROR("amdgpu: failed initialize dtn debugfs support.\n");
	1618	+ }
	1619	+#endif
	1620	+
	1621	+ for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
	1622	+ struct amdgpu_ring *ring = adev->rings[i];
	1623	+
	1624	+ if (!ring)
	1625	+ continue;
	1626	+
	1627	+ if (amdgpu_debugfs_ring_init(adev, ring)) {
	1628	+ DRM_ERROR("Failed to register debugfs file for rings !\n");
	1629	+ }
	1630	+ }
	1631	+
	1632	+ amdgpu_ras_debugfs_create_all(adev);
	1633	+
	1634	+ amdgpu_debugfs_autodump_init(adev);
	1635	+
	1636	+ amdgpu_rap_debugfs_init(adev);
	1637	+
937	1638	return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_list,
938	1639	ARRAY_SIZE(amdgpu_debugfs_list));
939	1640	}
..	..	@@ -947,5 +1648,4 @@
947	1648	{
948	1649	return 0;
949	1650	}
950		-void amdgpu_debugfs_regs_cleanup(struct amdgpu_device *adev) { }
951	1651	#endif