~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,13 +1,36 @@
1	1	// SPDX-License-Identifier: GPL-2.0
2		-/* Copyright (c) 2017-2018 The Linux Foundation. All rights reserved. */
	2	+/* Copyright (c) 2017-2019 The Linux Foundation. All rights reserved. */
3	3
4	4	#include <linux/clk.h>
5		-#include <linux/iopoll.h>
	5	+#include <linux/interconnect.h>
	6	+#include <linux/pm_domain.h>
6	7	#include <linux/pm_opp.h>
7	8	#include <soc/qcom/cmd-db.h>
	9	+#include <drm/drm_gem.h>
8	10
9	11	#include "a6xx_gpu.h"
10	12	#include "a6xx_gmu.xml.h"
	13	+#include "msm_gem.h"
	14	+#include "msm_gpu_trace.h"
	15	+#include "msm_mmu.h"
	16	+
	17	+static void a6xx_gmu_fault(struct a6xx_gmu *gmu)
	18	+{
	19	+ struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
	20	+ struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
	21	+ struct msm_gpu *gpu = &adreno_gpu->base;
	22	+ struct drm_device *dev = gpu->dev;
	23	+ struct msm_drm_private *priv = dev->dev_private;
	24	+
	25	+ /* FIXME: add a banner here */
	26	+ gmu->hung = true;
	27	+
	28	+ /* Turn off the hangcheck timer while we are resetting */
	29	+ del_timer(&gpu->hangcheck_timer);
	30	+
	31	+ /* Queue the GPU handler because we need to treat this as a recovery */
	32	+ queue_work(priv->wq, &gpu->recover_work);
	33	+}
11	34
12	35	static irqreturn_t a6xx_gmu_irq(int irq, void *data)
13	36	{
..	..	@@ -20,8 +43,7 @@
20	43	if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE) {
21	44	dev_err_ratelimited(gmu->dev, "GMU watchdog expired\n");
22	45
23		- /* Temporary until we can recover safely */
24		- BUG();
	46	+ a6xx_gmu_fault(gmu);
25	47	}
26	48
27	49	if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR)
..	..	@@ -42,35 +64,87 @@
42	64	status = gmu_read(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO);
43	65	gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, status);
44	66
45		- if (status & A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ)
46		- tasklet_schedule(&gmu->hfi_tasklet);
47		-
48	67	if (status & A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT) {
49	68	dev_err_ratelimited(gmu->dev, "GMU firmware fault\n");
50	69
51		- /* Temporary until we can recover safely */
52		- BUG();
	70	+ a6xx_gmu_fault(gmu);
53	71	}
54	72
55	73	return IRQ_HANDLED;
56	74	}
57	75
58		-/* Check to see if the GX rail is still powered */
59		-static bool a6xx_gmu_gx_is_on(struct a6xx_gmu *gmu)
	76	+bool a6xx_gmu_sptprac_is_on(struct a6xx_gmu *gmu)
60	77	{
61		- u32 val = gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS);
	78	+ u32 val;
	79	+
	80	+ /* This can be called from gpu state code so make sure GMU is valid */
	81	+ if (!gmu->initialized)
	82	+ return false;
	83	+
	84	+ val = gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS);
	85	+
	86	+ return !(val &
	87	+ (A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SPTPRAC_GDSC_POWER_OFF \|
	88	+ A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SP_CLOCK_OFF));
	89	+}
	90	+
	91	+/* Check to see if the GX rail is still powered */
	92	+bool a6xx_gmu_gx_is_on(struct a6xx_gmu *gmu)
	93	+{
	94	+ u32 val;
	95	+
	96	+ /* This can be called from gpu state code so make sure GMU is valid */
	97	+ if (!gmu->initialized)
	98	+ return false;
	99	+
	100	+ val = gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS);
62	101
63	102	return !(val &
64	103	(A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_GDSC_POWER_OFF \|
65	104	A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_CLK_OFF));
66	105	}
67	106
68		-static int a6xx_gmu_set_freq(struct a6xx_gmu *gmu, int index)
	107	+void a6xx_gmu_set_freq(struct msm_gpu gpu, struct dev_pm_opp opp)
69	108	{
	109	+ struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
	110	+ struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
	111	+ struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
	112	+ u32 perf_index;
	113	+ unsigned long gpu_freq;
	114	+ int ret = 0;
	115	+
	116	+ gpu_freq = dev_pm_opp_get_freq(opp);
	117	+
	118	+ if (gpu_freq == gmu->freq)
	119	+ return;
	120	+
	121	+ for (perf_index = 0; perf_index < gmu->nr_gpu_freqs - 1; perf_index++)
	122	+ if (gpu_freq == gmu->gpu_freqs[perf_index])
	123	+ break;
	124	+
	125	+ gmu->current_perf_index = perf_index;
	126	+ gmu->freq = gmu->gpu_freqs[perf_index];
	127	+
	128	+ trace_msm_gmu_freq_change(gmu->freq, perf_index);
	129	+
	130	+ /*
	131	+ * This can get called from devfreq while the hardware is idle. Don't
	132	+ * bring up the power if it isn't already active
	133	+ */
	134	+ if (pm_runtime_get_if_in_use(gmu->dev) == 0)
	135	+ return;
	136	+
	137	+ if (!gmu->legacy) {
	138	+ a6xx_hfi_set_freq(gmu, perf_index);
	139	+ dev_pm_opp_set_bw(&gpu->pdev->dev, opp);
	140	+ pm_runtime_put(gmu->dev);
	141	+ return;
	142	+ }
	143	+
70	144	gmu_write(gmu, REG_A6XX_GMU_DCVS_ACK_OPTION, 0);
71	145
72	146	gmu_write(gmu, REG_A6XX_GMU_DCVS_PERF_SETTING,
73		- ((index << 24) & 0xff) \| (3 & 0xf));
	147	+ ((3 & 0xf) << 28) \| perf_index);
74	148
75	149	/*
76	150	* Send an invalid index as a vote for the bus bandwidth and let the
..	..	@@ -82,7 +156,21 @@
82	156	a6xx_gmu_set_oob(gmu, GMU_OOB_DCVS_SET);
83	157	a6xx_gmu_clear_oob(gmu, GMU_OOB_DCVS_SET);
84	158
85		- return gmu_read(gmu, REG_A6XX_GMU_DCVS_RETURN);
	159	+ ret = gmu_read(gmu, REG_A6XX_GMU_DCVS_RETURN);
	160	+ if (ret)
	161	+ dev_err(gmu->dev, "GMU set GPU frequency error: %d\n", ret);
	162	+
	163	+ dev_pm_opp_set_bw(&gpu->pdev->dev, opp);
	164	+ pm_runtime_put(gmu->dev);
	165	+}
	166	+
	167	+unsigned long a6xx_gmu_get_freq(struct msm_gpu *gpu)
	168	+{
	169	+ struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
	170	+ struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
	171	+ struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
	172	+
	173	+ return gmu->freq;
86	174	}
87	175
88	176	static bool a6xx_gmu_check_idle_level(struct a6xx_gmu *gmu)
..	..	@@ -106,10 +194,8 @@
106	194	}
107	195
108	196	/* Wait for the GMU to get to its most idle state */
109		-int a6xx_gmu_wait_for_idle(struct a6xx_gpu *a6xx_gpu)
	197	+int a6xx_gmu_wait_for_idle(struct a6xx_gmu *gmu)
110	198	{
111		- struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
112		-
113	199	return spin_until(a6xx_gmu_check_idle_level(gmu));
114	200	}
115	201
..	..	@@ -129,13 +215,19 @@
129	215	}
130	216
131	217	gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1);
	218	+
	219	+ /* Set the log wptr index
	220	+ * note: downstream saves the value in poweroff and restores it here
	221	+ */
	222	+ gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_RESP, 0);
	223	+
132	224	gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 0);
133	225
134	226	ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, val,
135	227	(val & mask) == reset_val, 100, 10000);
136	228
137	229	if (ret)
138		- dev_err(gmu->dev, "GMU firmware initialization timed out\n");
	230	+ DRM_DEV_ERROR(gmu->dev, "GMU firmware initialization timed out\n");
139	231
140	232	return ret;
141	233	}
..	..	@@ -145,15 +237,12 @@
145	237	u32 val;
146	238	int ret;
147	239
148		- gmu_rmw(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK,
149		- A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ, 0);
150		-
151	240	gmu_write(gmu, REG_A6XX_GMU_HFI_CTRL_INIT, 1);
152	241
153	242	ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_HFI_CTRL_STATUS, val,
154	243	val & 1, 100, 10000);
155	244	if (ret)
156		- dev_err(gmu->dev, "Unable to start the HFI queues\n");
	245	+ DRM_DEV_ERROR(gmu->dev, "Unable to start the HFI queues\n");
157	246
158	247	return ret;
159	248	}
..	..	@@ -168,9 +257,24 @@
168	257
169	258	switch (state) {
170	259	case GMU_OOB_GPU_SET:
171		- request = GMU_OOB_GPU_SET_REQUEST;
172		- ack = GMU_OOB_GPU_SET_ACK;
	260	+ if (gmu->legacy) {
	261	+ request = GMU_OOB_GPU_SET_REQUEST;
	262	+ ack = GMU_OOB_GPU_SET_ACK;
	263	+ } else {
	264	+ request = GMU_OOB_GPU_SET_REQUEST_NEW;
	265	+ ack = GMU_OOB_GPU_SET_ACK_NEW;
	266	+ }
173	267	name = "GPU_SET";
	268	+ break;
	269	+ case GMU_OOB_PERFCOUNTER_SET:
	270	+ if (gmu->legacy) {
	271	+ request = GMU_OOB_PERFCOUNTER_REQUEST;
	272	+ ack = GMU_OOB_PERFCOUNTER_ACK;
	273	+ } else {
	274	+ request = GMU_OOB_PERFCOUNTER_REQUEST_NEW;
	275	+ ack = GMU_OOB_PERFCOUNTER_ACK_NEW;
	276	+ }
	277	+ name = "PERFCOUNTER";
174	278	break;
175	279	case GMU_OOB_BOOT_SLUMBER:
176	280	request = GMU_OOB_BOOT_SLUMBER_REQUEST;
..	..	@@ -194,7 +298,7 @@
194	298	val & (1 << ack), 100, 10000);
195	299
196	300	if (ret)
197		- dev_err(gmu->dev,
	301	+ DRM_DEV_ERROR(gmu->dev,
198	302	"Timeout waiting for GMU OOB set %s: 0x%x\n",
199	303	name,
200	304	gmu_read(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO));
..	..	@@ -208,10 +312,26 @@
208	312	/* Clear a pending OOB state in the GMU */
209	313	void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state)
210	314	{
	315	+ if (!gmu->legacy) {
	316	+ if (state == GMU_OOB_GPU_SET) {
	317	+ gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET,
	318	+ 1 << GMU_OOB_GPU_SET_CLEAR_NEW);
	319	+ } else {
	320	+ WARN_ON(state != GMU_OOB_PERFCOUNTER_SET);
	321	+ gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET,
	322	+ 1 << GMU_OOB_PERFCOUNTER_CLEAR_NEW);
	323	+ }
	324	+ return;
	325	+ }
	326	+
211	327	switch (state) {
212	328	case GMU_OOB_GPU_SET:
213	329	gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET,
214	330	1 << GMU_OOB_GPU_SET_CLEAR);
	331	+ break;
	332	+ case GMU_OOB_PERFCOUNTER_SET:
	333	+ gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET,
	334	+ 1 << GMU_OOB_PERFCOUNTER_CLEAR);
215	335	break;
216	336	case GMU_OOB_BOOT_SLUMBER:
217	337	gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET,
..	..	@@ -230,13 +350,16 @@
230	350	int ret;
231	351	u32 val;
232	352
	353	+ if (!gmu->legacy)
	354	+ return 0;
	355	+
233	356	gmu_write(gmu, REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL, 0x778000);
234	357
235	358	ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, val,
236	359	(val & 0x38) == 0x28, 1, 100);
237	360
238	361	if (ret) {
239		- dev_err(gmu->dev, "Unable to power on SPTPRAC: 0x%x\n",
	362	+ DRM_DEV_ERROR(gmu->dev, "Unable to power on SPTPRAC: 0x%x\n",
240	363	gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS));
241	364	}
242	365
..	..	@@ -249,6 +372,9 @@
249	372	u32 val;
250	373	int ret;
251	374
	375	+ if (!gmu->legacy)
	376	+ return;
	377	+
252	378	/* Make sure retention is on */
253	379	gmu_rmw(gmu, REG_A6XX_GPU_CC_GX_GDSCR, 0, (1 << 11));
254	380
..	..	@@ -258,7 +384,7 @@
258	384	(val & 0x04), 100, 10000);
259	385
260	386	if (ret)
261		- dev_err(gmu->dev, "failed to power off SPTPRAC: 0x%x\n",
	387	+ DRM_DEV_ERROR(gmu->dev, "failed to power off SPTPRAC: 0x%x\n",
262	388	gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS));
263	389	}
264	390
..	..	@@ -292,6 +418,11 @@
292	418	if (gmu->idle_level < GMU_IDLE_STATE_SPTP)
293	419	a6xx_sptprac_disable(gmu);
294	420
	421	+ if (!gmu->legacy) {
	422	+ ret = a6xx_hfi_send_prep_slumber(gmu);
	423	+ goto out;
	424	+ }
	425	+
295	426	/* Tell the GMU to get ready to slumber */
296	427	gmu_write(gmu, REG_A6XX_GMU_BOOT_SLUMBER_OPTION, 1);
297	428
..	..	@@ -302,11 +433,12 @@
302	433	/* Check to see if the GMU really did slumber */
303	434	if (gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE)
304	435	!= 0x0f) {
305		- dev_err(gmu->dev, "The GMU did not go into slumber\n");
	436	+ DRM_DEV_ERROR(gmu->dev, "The GMU did not go into slumber\n");
306	437	ret = -ETIMEDOUT;
307	438	}
308	439	}
309	440
	441	+out:
310	442	/* Put fence into allow mode */
311	443	gmu_write(gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, 0);
312	444	return ret;
..	..	@@ -324,23 +456,27 @@
324	456	ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_RSCC_CONTROL_ACK, val,
325	457	val & (1 << 1), 100, 10000);
326	458	if (ret) {
327		- dev_err(gmu->dev, "Unable to power on the GPU RSC\n");
	459	+ DRM_DEV_ERROR(gmu->dev, "Unable to power on the GPU RSC\n");
328	460	return ret;
329	461	}
330	462
331		- ret = gmu_poll_timeout(gmu, REG_A6XX_RSCC_SEQ_BUSY_DRV0, val,
	463	+ ret = gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_SEQ_BUSY_DRV0, val,
332	464	!val, 100, 10000);
333	465
334		- if (!ret) {
335		- gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0);
336		-
337		- /* Re-enable the power counter */
338		- gmu_write(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 1);
339		- return 0;
	466	+ if (ret) {
	467	+ DRM_DEV_ERROR(gmu->dev, "GPU RSC sequence stuck while waking up the GPU\n");
	468	+ return ret;
340	469	}
341	470
342		- dev_err(gmu->dev, "GPU RSC sequence stuck while waking up the GPU\n");
343		- return ret;
	471	+ gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0);
	472	+
	473	+ /* Set up CX GMU counter 0 to count busy ticks */
	474	+ gmu_write(gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK, 0xff000000);
	475	+ gmu_rmw(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0, 0xff, 0x20);
	476	+
	477	+ /* Enable the power counter */
	478	+ gmu_write(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 1);
	479	+ return 0;
344	480	}
345	481
346	482	static void a6xx_rpmh_stop(struct a6xx_gmu *gmu)
..	..	@@ -350,78 +486,123 @@
350	486
351	487	gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 1);
352	488
353		- ret = gmu_poll_timeout(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0,
	489	+ ret = gmu_poll_timeout_rscc(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0,
354	490	val, val & (1 << 16), 100, 10000);
355	491	if (ret)
356		- dev_err(gmu->dev, "Unable to power off the GPU RSC\n");
	492	+ DRM_DEV_ERROR(gmu->dev, "Unable to power off the GPU RSC\n");
357	493
358	494	gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0);
359	495	}
360	496
	497	+static inline void pdc_write(void __iomem *ptr, u32 offset, u32 value)
	498	+{
	499	+ return msm_writel(value, ptr + (offset << 2));
	500	+}
	501	+
	502	+static void __iomem a6xx_gmu_get_mmio(struct platform_device pdev,
	503	+ const char *name);
	504	+
361	505	static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu)
362	506	{
	507	+ struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
	508	+ struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
	509	+ struct platform_device *pdev = to_platform_device(gmu->dev);
	510	+ void __iomem *pdcptr = a6xx_gmu_get_mmio(pdev, "gmu_pdc");
	511	+ void __iomem *seqptr = a6xx_gmu_get_mmio(pdev, "gmu_pdc_seq");
	512	+ uint32_t pdc_address_offset;
	513	+
	514	+ if (!pdcptr \|\| !seqptr)
	515	+ goto err;
	516	+
	517	+ if (adreno_is_a618(adreno_gpu) \|\| adreno_is_a640(adreno_gpu))
	518	+ pdc_address_offset = 0x30090;
	519	+ else if (adreno_is_a650(adreno_gpu))
	520	+ pdc_address_offset = 0x300a0;
	521	+ else
	522	+ pdc_address_offset = 0x30080;
	523	+
363	524	/* Disable SDE clock gating */
364		- gmu_write(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, BIT(24));
	525	+ gmu_write_rscc(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, BIT(24));
365	526
366	527	/* Setup RSC PDC handshake for sleep and wakeup */
367		- gmu_write(gmu, REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0, 1);
368		- gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA, 0);
369		- gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR, 0);
370		- gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 2, 0);
371		- gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 2, 0);
372		- gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 4, 0x80000000);
373		- gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 4, 0);
374		- gmu_write(gmu, REG_A6XX_RSCC_OVERRIDE_START_ADDR, 0);
375		- gmu_write(gmu, REG_A6XX_RSCC_PDC_SEQ_START_ADDR, 0x4520);
376		- gmu_write(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_LO, 0x4510);
377		- gmu_write(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_HI, 0x4514);
	528	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0, 1);
	529	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA, 0);
	530	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR, 0);
	531	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 2, 0);
	532	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 2, 0);
	533	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 4, 0x80000000);
	534	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 4, 0);
	535	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_OVERRIDE_START_ADDR, 0);
	536	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_SEQ_START_ADDR, 0x4520);
	537	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_LO, 0x4510);
	538	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_HI, 0x4514);
378	539
379	540	/* Load RSC sequencer uCode for sleep and wakeup */
380		- gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0, 0xa7a506a0);
381		- gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 1, 0xa1e6a6e7);
382		- gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 2, 0xa2e081e1);
383		- gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 3, 0xe9a982e2);
384		- gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 4, 0x0020e8a8);
	541	+ if (adreno_is_a650(adreno_gpu)) {
	542	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0, 0xeaaae5a0);
	543	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 1, 0xe1a1ebab);
	544	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 2, 0xa2e0a581);
	545	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 3, 0xecac82e2);
	546	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 4, 0x0020edad);
	547	+ } else {
	548	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0, 0xa7a506a0);
	549	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 1, 0xa1e6a6e7);
	550	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 2, 0xa2e081e1);
	551	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 3, 0xe9a982e2);
	552	+ gmu_write_rscc(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 4, 0x0020e8a8);
	553	+ }
385	554
386	555	/* Load PDC sequencer uCode for power up and power down sequence */
387		- pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0, 0xfebea1e1);
388		- pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 1, 0xa5a4a3a2);
389		- pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 2, 0x8382a6e0);
390		- pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 3, 0xbce3e284);
391		- pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 4, 0x002081fc);
	556	+ pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0, 0xfebea1e1);
	557	+ pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 1, 0xa5a4a3a2);
	558	+ pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 2, 0x8382a6e0);
	559	+ pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 3, 0xbce3e284);
	560	+ pdc_write(seqptr, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 4, 0x002081fc);
392	561
393	562	/* Set TCS commands used by PDC sequence for low power modes */
394		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD_ENABLE_BANK, 7);
395		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD_WAIT_FOR_CMPL_BANK, 0);
396		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CONTROL, 0);
397		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID, 0x10108);
398		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR, 0x30010);
399		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA, 1);
400		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 4, 0x10108);
401		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 4, 0x30000);
402		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 4, 0x0);
403		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 8, 0x10108);
404		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 8, 0x30080);
405		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 8, 0x0);
406		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD_ENABLE_BANK, 7);
407		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD_WAIT_FOR_CMPL_BANK, 0);
408		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CONTROL, 0);
409		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID, 0x10108);
410		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR, 0x30010);
411		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA, 2);
412		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 4, 0x10108);
413		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 4, 0x30000);
414		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 4, 0x3);
415		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 8, 0x10108);
416		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 8, 0x30080);
417		- pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 8, 0x3);
	563	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD_ENABLE_BANK, 7);
	564	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD_WAIT_FOR_CMPL_BANK, 0);
	565	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CONTROL, 0);
	566	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID, 0x10108);
	567	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR, 0x30010);
	568	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA, 1);
	569	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 4, 0x10108);
	570	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 4, 0x30000);
	571	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 4, 0x0);
	572	+
	573	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 8, 0x10108);
	574	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 8, pdc_address_offset);
	575	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 8, 0x0);
	576	+
	577	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD_ENABLE_BANK, 7);
	578	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD_WAIT_FOR_CMPL_BANK, 0);
	579	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CONTROL, 0);
	580	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID, 0x10108);
	581	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR, 0x30010);
	582	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA, 2);
	583	+
	584	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 4, 0x10108);
	585	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 4, 0x30000);
	586	+ if (adreno_is_a618(adreno_gpu) \|\| adreno_is_a650(adreno_gpu))
	587	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 4, 0x2);
	588	+ else
	589	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 4, 0x3);
	590	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 8, 0x10108);
	591	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 8, pdc_address_offset);
	592	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 8, 0x3);
418	593
419	594	/* Setup GPU PDC */
420		- pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_START_ADDR, 0);
421		- pdc_write(gmu, REG_A6XX_PDC_GPU_ENABLE_PDC, 0x80000001);
	595	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_SEQ_START_ADDR, 0);
	596	+ pdc_write(pdcptr, REG_A6XX_PDC_GPU_ENABLE_PDC, 0x80000001);
422	597
423	598	/* ensure no writes happen before the uCode is fully written */
424	599	wmb();
	600	+
	601	+err:
	602	+ if (!IS_ERR_OR_NULL(pdcptr))
	603	+ iounmap(pdcptr);
	604	+ if (!IS_ERR_OR_NULL(seqptr))
	605	+ iounmap(seqptr);
425	606	}
426	607
427	608	/*
..	..	@@ -437,6 +618,8 @@
437	618	{
438	619	/* Disable GMU WB/RB buffer */
439	620	gmu_write(gmu, REG_A6XX_GMU_SYS_BUS_CONFIG, 0x1);
	621	+ gmu_write(gmu, REG_A6XX_GMU_ICACHE_CONFIG, 0x1);
	622	+ gmu_write(gmu, REG_A6XX_GMU_DCACHE_CONFIG, 0x1);
440	623
441	624	gmu_write(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0x9c40400);
442	625
..	..	@@ -447,7 +630,7 @@
447	630	gmu_rmw(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0,
448	631	A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_IFPC_ENABLE \|
449	632	A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_HM_POWER_COLLAPSE_ENABLE);
450		- /* Fall through */
	633	+ fallthrough;
451	634	case GMU_IDLE_STATE_SPTP:
452	635	gmu_write(gmu, REG_A6XX_GMU_PWR_COL_SPTPRAC_HYST,
453	636	GMU_PWR_COL_HYST);
..	..	@@ -466,14 +649,95 @@
466	649	A6XX_GMU_RPMH_CTRL_GFX_VOTE_ENABLE);
467	650	}
468	651
	652	+struct block_header {
	653	+ u32 addr;
	654	+ u32 size;
	655	+ u32 type;
	656	+ u32 value;
	657	+ u32 data[];
	658	+};
	659	+
	660	+/* this should be a general kernel helper */
	661	+static int in_range(u32 addr, u32 start, u32 size)
	662	+{
	663	+ return addr >= start && addr < start + size;
	664	+}
	665	+
	666	+static bool fw_block_mem(struct a6xx_gmu_bo bo, const struct block_header blk)
	667	+{
	668	+ if (!in_range(blk->addr, bo->iova, bo->size))
	669	+ return false;
	670	+
	671	+ memcpy(bo->virt + blk->addr - bo->iova, blk->data, blk->size);
	672	+ return true;
	673	+}
	674	+
	675	+static int a6xx_gmu_fw_load(struct a6xx_gmu *gmu)
	676	+{
	677	+ struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
	678	+ struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
	679	+ const struct firmware *fw_image = adreno_gpu->fw[ADRENO_FW_GMU];
	680	+ const struct block_header *blk;
	681	+ u32 reg_offset;
	682	+
	683	+ u32 itcm_base = 0x00000000;
	684	+ u32 dtcm_base = 0x00040000;
	685	+
	686	+ if (adreno_is_a650(adreno_gpu))
	687	+ dtcm_base = 0x10004000;
	688	+
	689	+ if (gmu->legacy) {
	690	+ /* Sanity check the size of the firmware that was loaded */
	691	+ if (fw_image->size > 0x8000) {
	692	+ DRM_DEV_ERROR(gmu->dev,
	693	+ "GMU firmware is bigger than the available region\n");
	694	+ return -EINVAL;
	695	+ }
	696	+
	697	+ gmu_write_bulk(gmu, REG_A6XX_GMU_CM3_ITCM_START,
	698	+ (u32*) fw_image->data, fw_image->size);
	699	+ return 0;
	700	+ }
	701	+
	702	+
	703	+ for (blk = (const struct block_header *) fw_image->data;
	704	+ (const u8*) blk < fw_image->data + fw_image->size;
	705	+ blk = (const struct block_header *) &blk->data[blk->size >> 2]) {
	706	+ if (blk->size == 0)
	707	+ continue;
	708	+
	709	+ if (in_range(blk->addr, itcm_base, SZ_16K)) {
	710	+ reg_offset = (blk->addr - itcm_base) >> 2;
	711	+ gmu_write_bulk(gmu,
	712	+ REG_A6XX_GMU_CM3_ITCM_START + reg_offset,
	713	+ blk->data, blk->size);
	714	+ } else if (in_range(blk->addr, dtcm_base, SZ_16K)) {
	715	+ reg_offset = (blk->addr - dtcm_base) >> 2;
	716	+ gmu_write_bulk(gmu,
	717	+ REG_A6XX_GMU_CM3_DTCM_START + reg_offset,
	718	+ blk->data, blk->size);
	719	+ } else if (!fw_block_mem(&gmu->icache, blk) &&
	720	+ !fw_block_mem(&gmu->dcache, blk) &&
	721	+ !fw_block_mem(&gmu->dummy, blk)) {
	722	+ DRM_DEV_ERROR(gmu->dev,
	723	+ "failed to match fw block (addr=%.8x size=%d data[0]=%.8x)\n",
	724	+ blk->addr, blk->size, blk->data[0]);
	725	+ }
	726	+ }
	727	+
	728	+ return 0;
	729	+}
	730	+
469	731	static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state)
470	732	{
471	733	static bool rpmh_init;
472	734	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
473	735	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
474		- int i, ret;
	736	+ int ret;
475	737	u32 chipid;
476		- u32 *image;
	738	+
	739	+ if (adreno_is_a650(adreno_gpu))
	740	+ gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_CX_FAL_INTF, 1);
477	741
478	742	if (state == GMU_WARM_BOOT) {
479	743	ret = a6xx_rpmh_start(gmu);
..	..	@@ -484,13 +748,6 @@
484	748	"GMU firmware is not loaded\n"))
485	749	return -ENOENT;
486	750
487		- /* Sanity check the size of the firmware that was loaded */
488		- if (adreno_gpu->fw[ADRENO_FW_GMU]->size > 0x8000) {
489		- dev_err(gmu->dev,
490		- "GMU firmware is bigger than the available region\n");
491		- return -EINVAL;
492		- }
493		-
494	751	/* Turn on register retention */
495	752	gmu_write(gmu, REG_A6XX_GMU_GENERAL_7, 1);
496	753
..	..	@@ -498,24 +755,22 @@
498	755	if (!rpmh_init) {
499	756	a6xx_gmu_rpmh_init(gmu);
500	757	rpmh_init = true;
501		- } else if (state != GMU_RESET) {
	758	+ } else {
502	759	ret = a6xx_rpmh_start(gmu);
503	760	if (ret)
504	761	return ret;
505	762	}
506	763
507		- image = (u32 *) adreno_gpu->fw[ADRENO_FW_GMU]->data;
508		-
509		- for (i = 0; i < adreno_gpu->fw[ADRENO_FW_GMU]->size >> 2; i++)
510		- gmu_write(gmu, REG_A6XX_GMU_CM3_ITCM_START + i,
511		- image[i]);
	764	+ ret = a6xx_gmu_fw_load(gmu);
	765	+ if (ret)
	766	+ return ret;
512	767	}
513	768
514	769	gmu_write(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, 0);
515	770	gmu_write(gmu, REG_A6XX_GMU_CM3_BOOT_CONFIG, 0x02);
516	771
517	772	/* Write the iova of the HFI table */
518		- gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_ADDR, gmu->hfi->iova);
	773	+ gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_ADDR, gmu->hfi.iova);
519	774	gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_INFO, 1);
520	775
521	776	gmu_write(gmu, REG_A6XX_GMU_AHB_FENCE_RANGE_0,
..	..	@@ -528,6 +783,9 @@
528	783
529	784	gmu_write(gmu, REG_A6XX_GMU_HFI_SFR_ADDR, chipid);
530	785
	786	+ gmu_write(gmu, REG_A6XX_GPU_GMU_CX_GMU_PWR_COL_CP_MSG,
	787	+ gmu->log.iova \| (gmu->log.size / SZ_4K - 1));
	788	+
531	789	/* Set up the lowest idle level on the GMU */
532	790	a6xx_gmu_power_config(gmu);
533	791
..	..	@@ -535,9 +793,11 @@
535	793	if (ret)
536	794	return ret;
537	795
538		- ret = a6xx_gmu_gfx_rail_on(gmu);
539		- if (ret)
540		- return ret;
	796	+ if (gmu->legacy) {
	797	+ ret = a6xx_gmu_gfx_rail_on(gmu);
	798	+ if (ret)
	799	+ return ret;
	800	+ }
541	801
542	802	/* Enable SPTP_PC if the CPU is responsible for it */
543	803	if (gmu->idle_level < GMU_IDLE_STATE_SPTP) {
..	..	@@ -557,27 +817,12 @@
557	817	}
558	818
559	819	#define A6XX_HFI_IRQ_MASK \
560		- (A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ \| \
561		- A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT)
	820	+ (A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT)
562	821
563	822	#define A6XX_GMU_IRQ_MASK \
564	823	(A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE \| \
565	824	A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR \| \
566	825	A6XX_GMU_AO_HOST_INTERRUPT_STATUS_FENCE_ERR)
567		-
568		-static void a6xx_gmu_irq_enable(struct a6xx_gmu *gmu)
569		-{
570		- gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, ~0);
571		- gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, ~0);
572		-
573		- gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK,
574		- ~A6XX_GMU_IRQ_MASK);
575		- gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK,
576		- ~A6XX_HFI_IRQ_MASK);
577		-
578		- enable_irq(gmu->gmu_irq);
579		- enable_irq(gmu->hfi_irq);
580		-}
581	826
582	827	static void a6xx_gmu_irq_disable(struct a6xx_gmu *gmu)
583	828	{
..	..	@@ -588,12 +833,24 @@
588	833	gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK, ~0);
589	834	}
590	835
591		-int a6xx_gmu_reset(struct a6xx_gpu *a6xx_gpu)
	836	+static void a6xx_gmu_rpmh_off(struct a6xx_gmu *gmu)
592	837	{
593		- struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
594		- int ret;
595	838	u32 val;
596	839
	840	+ /* Make sure there are no outstanding RPMh votes */
	841	+ gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS0_DRV0_STATUS, val,
	842	+ (val & 1), 100, 10000);
	843	+ gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS1_DRV0_STATUS, val,
	844	+ (val & 1), 100, 10000);
	845	+ gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS2_DRV0_STATUS, val,
	846	+ (val & 1), 100, 10000);
	847	+ gmu_poll_timeout_rscc(gmu, REG_A6XX_RSCC_TCS3_DRV0_STATUS, val,
	848	+ (val & 1), 100, 1000);
	849	+}
	850	+
	851	+/* Force the GMU off in case it isn't responsive */
	852	+static void a6xx_gmu_force_off(struct a6xx_gmu *gmu)
	853	+{
597	854	/* Flush all the queues */
598	855	a6xx_hfi_stop(gmu);
599	856
..	..	@@ -604,83 +861,114 @@
604	861	a6xx_sptprac_disable(gmu);
605	862
606	863	/* Make sure there are no outstanding RPMh votes */
607		- gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS0_DRV0_STATUS, val,
608		- (val & 1), 100, 10000);
609		- gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS1_DRV0_STATUS, val,
610		- (val & 1), 100, 10000);
611		- gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS2_DRV0_STATUS, val,
612		- (val & 1), 100, 10000);
613		- gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS3_DRV0_STATUS, val,
614		- (val & 1), 100, 1000);
	864	+ a6xx_gmu_rpmh_off(gmu);
	865	+}
615	866
616		- /* Force off the GX GSDC */
617		- regulator_force_disable(gmu->gx);
	867	+static void a6xx_gmu_set_initial_freq(struct msm_gpu gpu, struct a6xx_gmu gmu)
	868	+{
	869	+ struct dev_pm_opp *gpu_opp;
	870	+ unsigned long gpu_freq = gmu->gpu_freqs[gmu->current_perf_index];
618	871
619		- /* Disable the resources */
620		- clk_bulk_disable_unprepare(gmu->nr_clocks, gmu->clocks);
621		- pm_runtime_put_sync(gmu->dev);
	872	+ gpu_opp = dev_pm_opp_find_freq_exact(&gpu->pdev->dev, gpu_freq, true);
	873	+ if (IS_ERR_OR_NULL(gpu_opp))
	874	+ return;
622	875
623		- /* Re-enable the resources */
624		- pm_runtime_get_sync(gmu->dev);
	876	+ gmu->freq = 0; /* so a6xx_gmu_set_freq() doesn't exit early */
	877	+ a6xx_gmu_set_freq(gpu, gpu_opp);
	878	+ dev_pm_opp_put(gpu_opp);
	879	+}
625	880
626		- /* Use a known rate to bring up the GMU */
627		- clk_set_rate(gmu->core_clk, 200000000);
628		- ret = clk_bulk_prepare_enable(gmu->nr_clocks, gmu->clocks);
629		- if (ret)
630		- goto out;
	881	+static void a6xx_gmu_set_initial_bw(struct msm_gpu gpu, struct a6xx_gmu gmu)
	882	+{
	883	+ struct dev_pm_opp *gpu_opp;
	884	+ unsigned long gpu_freq = gmu->gpu_freqs[gmu->current_perf_index];
631	885
632		- a6xx_gmu_irq_enable(gmu);
	886	+ gpu_opp = dev_pm_opp_find_freq_exact(&gpu->pdev->dev, gpu_freq, true);
	887	+ if (IS_ERR_OR_NULL(gpu_opp))
	888	+ return;
633	889
634		- ret = a6xx_gmu_fw_start(gmu, GMU_RESET);
635		- if (!ret)
636		- ret = a6xx_hfi_start(gmu, GMU_COLD_BOOT);
637		-
638		- /* Set the GPU back to the highest power frequency */
639		- a6xx_gmu_set_freq(gmu, gmu->nr_gpu_freqs - 1);
640		-
641		-out:
642		- if (ret)
643		- a6xx_gmu_clear_oob(gmu, GMU_OOB_BOOT_SLUMBER);
644		-
645		- return ret;
	890	+ dev_pm_opp_set_bw(&gpu->pdev->dev, gpu_opp);
	891	+ dev_pm_opp_put(gpu_opp);
646	892	}
647	893
648	894	int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu)
649	895	{
	896	+ struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
	897	+ struct msm_gpu *gpu = &adreno_gpu->base;
650	898	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
651	899	int status, ret;
652	900
653		- if (WARN(!gmu->mmio, "The GMU is not set up yet\n"))
	901	+ if (WARN(!gmu->initialized, "The GMU is not set up yet\n"))
654	902	return 0;
	903	+
	904	+ gmu->hung = false;
655	905
656	906	/* Turn on the resources */
657	907	pm_runtime_get_sync(gmu->dev);
658	908
	909	+ /*
	910	+ * "enable" the GX power domain which won't actually do anything but it
	911	+ * will make sure that the refcounting is correct in case we need to
	912	+ * bring down the GX after a GMU failure
	913	+ */
	914	+ if (!IS_ERR_OR_NULL(gmu->gxpd))
	915	+ pm_runtime_get_sync(gmu->gxpd);
	916	+
659	917	/* Use a known rate to bring up the GMU */
660	918	clk_set_rate(gmu->core_clk, 200000000);
661	919	ret = clk_bulk_prepare_enable(gmu->nr_clocks, gmu->clocks);
662		- if (ret)
663		- goto out;
	920	+ if (ret) {
	921	+ pm_runtime_put(gmu->gxpd);
	922	+ pm_runtime_put(gmu->dev);
	923	+ return ret;
	924	+ }
664	925
665		- a6xx_gmu_irq_enable(gmu);
	926	+ /* Set the bus quota to a reasonable value for boot */
	927	+ a6xx_gmu_set_initial_bw(gpu, gmu);
	928	+
	929	+ /* Enable the GMU interrupt */
	930	+ gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, ~0);
	931	+ gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK, ~A6XX_GMU_IRQ_MASK);
	932	+ enable_irq(gmu->gmu_irq);
666	933
667	934	/* Check to see if we are doing a cold or warm boot */
668	935	status = gmu_read(gmu, REG_A6XX_GMU_GENERAL_7) == 1 ?
669	936	GMU_WARM_BOOT : GMU_COLD_BOOT;
	937	+
	938	+ /*
	939	+ * Warm boot path does not work on newer GPUs
	940	+ * Presumably this is because icache/dcache regions must be restored
	941	+ */
	942	+ if (!gmu->legacy)
	943	+ status = GMU_COLD_BOOT;
670	944
671	945	ret = a6xx_gmu_fw_start(gmu, status);
672	946	if (ret)
673	947	goto out;
674	948
675	949	ret = a6xx_hfi_start(gmu, status);
	950	+ if (ret)
	951	+ goto out;
676	952
677		- /* Set the GPU to the highest power frequency */
678		- a6xx_gmu_set_freq(gmu, gmu->nr_gpu_freqs - 1);
	953	+ /*
	954	+ * Turn on the GMU firmware fault interrupt after we know the boot
	955	+ * sequence is successful
	956	+ */
	957	+ gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, ~0);
	958	+ gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK, ~A6XX_HFI_IRQ_MASK);
	959	+ enable_irq(gmu->hfi_irq);
	960	+
	961	+ /* Set the GPU to the current freq */
	962	+ a6xx_gmu_set_initial_freq(gpu, gmu);
679	963
680	964	out:
681		- /* Make sure to turn off the boot OOB request on error */
682		- if (ret)
683		- a6xx_gmu_clear_oob(gmu, GMU_OOB_BOOT_SLUMBER);
	965	+ /* On failure, shut down the GMU to leave it in a good state */
	966	+ if (ret) {
	967	+ disable_irq(gmu->gmu_irq);
	968	+ a6xx_rpmh_stop(gmu);
	969	+ pm_runtime_put(gmu->gxpd);
	970	+ pm_runtime_put(gmu->dev);
	971	+ }
684	972
685	973	return ret;
686	974	}
..	..	@@ -689,7 +977,7 @@
689	977	{
690	978	u32 reg;
691	979
692		- if (!gmu->mmio)
	980	+ if (!gmu->initialized)
693	981	return true;
694	982
695	983	reg = gmu_read(gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS);
..	..	@@ -700,9 +988,41 @@
700	988	return true;
701	989	}
702	990
703		-int a6xx_gmu_stop(struct a6xx_gpu *a6xx_gpu)
	991	+#define GBIF_CLIENT_HALT_MASK BIT(0)
	992	+#define GBIF_ARB_HALT_MASK BIT(1)
	993	+
	994	+static void a6xx_bus_clear_pending_transactions(struct adreno_gpu *adreno_gpu)
704	995	{
705		- struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
	996	+ struct msm_gpu *gpu = &adreno_gpu->base;
	997	+
	998	+ if (!a6xx_has_gbif(adreno_gpu)) {
	999	+ gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0xf);
	1000	+ spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) &
	1001	+ 0xf) == 0xf);
	1002	+ gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0);
	1003	+
	1004	+ return;
	1005	+ }
	1006	+
	1007	+ /* Halt new client requests on GBIF */
	1008	+ gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_CLIENT_HALT_MASK);
	1009	+ spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) &
	1010	+ (GBIF_CLIENT_HALT_MASK)) == GBIF_CLIENT_HALT_MASK);
	1011	+
	1012	+ /* Halt all AXI requests on GBIF */
	1013	+ gpu_write(gpu, REG_A6XX_GBIF_HALT, GBIF_ARB_HALT_MASK);
	1014	+ spin_until((gpu_read(gpu, REG_A6XX_GBIF_HALT_ACK) &
	1015	+ (GBIF_ARB_HALT_MASK)) == GBIF_ARB_HALT_MASK);
	1016	+
	1017	+ /* The GBIF halt needs to be explicitly cleared */
	1018	+ gpu_write(gpu, REG_A6XX_GBIF_HALT, 0x0);
	1019	+}
	1020	+
	1021	+/* Gracefully try to shut down the GMU and by extension the GPU */
	1022	+static void a6xx_gmu_shutdown(struct a6xx_gmu *gmu)
	1023	+{
	1024	+ struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
	1025	+ struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
706	1026	u32 val;
707	1027
708	1028	/*
..	..	@@ -712,10 +1032,15 @@
712	1032	val = gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE);
713	1033
714	1034	if (val != 0xf) {
715		- int ret = a6xx_gmu_wait_for_idle(a6xx_gpu);
	1035	+ int ret = a6xx_gmu_wait_for_idle(gmu);
716	1036
717		- /* Temporary until we can recover safely */
718		- BUG_ON(ret);
	1037	+ /* If the GMU isn't responding assume it is hung */
	1038	+ if (ret) {
	1039	+ a6xx_gmu_force_off(gmu);
	1040	+ return;
	1041	+ }
	1042	+
	1043	+ a6xx_bus_clear_pending_transactions(adreno_gpu);
719	1044
720	1045	/* tell the GMU we want to slumber */
721	1046	a6xx_gmu_notify_slumber(gmu);
..	..	@@ -731,7 +1056,7 @@
731	1056	*/
732	1057
733	1058	if (ret)
734		- dev_err(gmu->dev,
	1059	+ DRM_DEV_ERROR(gmu->dev,
735	1060	"Unable to slumber GMU: status = 0%x/0%x\n",
736	1061	gmu_read(gmu,
737	1062	REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS),
..	..	@@ -747,6 +1072,36 @@
747	1072
748	1073	/* Tell RPMh to power off the GPU */
749	1074	a6xx_rpmh_stop(gmu);
	1075	+}
	1076	+
	1077	+
	1078	+int a6xx_gmu_stop(struct a6xx_gpu *a6xx_gpu)
	1079	+{
	1080	+ struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
	1081	+ struct msm_gpu *gpu = &a6xx_gpu->base.base;
	1082	+
	1083	+ if (!pm_runtime_active(gmu->dev))
	1084	+ return 0;
	1085	+
	1086	+ /*
	1087	+ * Force the GMU off if we detected a hang, otherwise try to shut it
	1088	+ * down gracefully
	1089	+ */
	1090	+ if (gmu->hung)
	1091	+ a6xx_gmu_force_off(gmu);
	1092	+ else
	1093	+ a6xx_gmu_shutdown(gmu);
	1094	+
	1095	+ /* Remove the bus vote */
	1096	+ dev_pm_opp_set_bw(&gpu->pdev->dev, NULL);
	1097	+
	1098	+ /*
	1099	+ * Make sure the GX domain is off before turning off the GMU (CX)
	1100	+ * domain. Usually the GMU does this but only if the shutdown sequence
	1101	+ * was successful
	1102	+ */
	1103	+ if (!IS_ERR_OR_NULL(gmu->gxpd))
	1104	+ pm_runtime_put_sync(gmu->gxpd);
750	1105
751	1106	clk_bulk_disable_unprepare(gmu->nr_clocks, gmu->clocks);
752	1107
..	..	@@ -755,160 +1110,92 @@
755	1110	return 0;
756	1111	}
757	1112
758		-static void a6xx_gmu_memory_free(struct a6xx_gmu gmu, struct a6xx_gmu_bo bo)
	1113	+static void a6xx_gmu_memory_free(struct a6xx_gmu *gmu)
759	1114	{
760		- int count, i;
761		- u64 iova;
	1115	+ msm_gem_kernel_put(gmu->hfi.obj, gmu->aspace, false);
	1116	+ msm_gem_kernel_put(gmu->debug.obj, gmu->aspace, false);
	1117	+ msm_gem_kernel_put(gmu->icache.obj, gmu->aspace, false);
	1118	+ msm_gem_kernel_put(gmu->dcache.obj, gmu->aspace, false);
	1119	+ msm_gem_kernel_put(gmu->dummy.obj, gmu->aspace, false);
	1120	+ msm_gem_kernel_put(gmu->log.obj, gmu->aspace, false);
762	1121
763		- if (IS_ERR_OR_NULL(bo))
764		- return;
765		-
766		- count = bo->size >> PAGE_SHIFT;
767		- iova = bo->iova;
768		-
769		- for (i = 0; i < count; i++, iova += PAGE_SIZE) {
770		- iommu_unmap(gmu->domain, iova, PAGE_SIZE);
771		- __free_pages(bo->pages[i], 0);
772		- }
773		-
774		- kfree(bo->pages);
775		- kfree(bo);
	1122	+ gmu->aspace->mmu->funcs->detach(gmu->aspace->mmu);
	1123	+ msm_gem_address_space_put(gmu->aspace);
776	1124	}
777	1125
778		-static struct a6xx_gmu_bo a6xx_gmu_memory_alloc(struct a6xx_gmu gmu,
779		- size_t size)
	1126	+static int a6xx_gmu_memory_alloc(struct a6xx_gmu gmu, struct a6xx_gmu_bo bo,
	1127	+ size_t size, u64 iova)
780	1128	{
781		- struct a6xx_gmu_bo *bo;
782		- int ret, count, i;
	1129	+ struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
	1130	+ struct drm_device *dev = a6xx_gpu->base.base.dev;
	1131	+ uint32_t flags = MSM_BO_WC;
	1132	+ u64 range_start, range_end;
	1133	+ int ret;
783	1134
784		- bo = kzalloc(sizeof(*bo), GFP_KERNEL);
785		- if (!bo)
786		- return ERR_PTR(-ENOMEM);
787		-
788		- bo->size = PAGE_ALIGN(size);
789		-
790		- count = bo->size >> PAGE_SHIFT;
791		-
792		- bo->pages = kcalloc(count, sizeof(struct page *), GFP_KERNEL);
793		- if (!bo->pages) {
794		- kfree(bo);
795		- return ERR_PTR(-ENOMEM);
	1135	+ size = PAGE_ALIGN(size);
	1136	+ if (!iova) {
	1137	+ /* no fixed address - use GMU's uncached range */
	1138	+ range_start = 0x60000000 + PAGE_SIZE; /* skip dummy page */
	1139	+ range_end = 0x80000000;
	1140	+ } else {
	1141	+ /* range for fixed address */
	1142	+ range_start = iova;
	1143	+ range_end = iova + size;
	1144	+ /* use IOMMU_PRIV for icache/dcache */
	1145	+ flags \|= MSM_BO_MAP_PRIV;
796	1146	}
797	1147
798		- for (i = 0; i < count; i++) {
799		- bo->pages[i] = alloc_page(GFP_KERNEL);
800		- if (!bo->pages[i])
801		- goto err;
	1148	+ bo->obj = msm_gem_new(dev, size, flags);
	1149	+ if (IS_ERR(bo->obj))
	1150	+ return PTR_ERR(bo->obj);
	1151	+
	1152	+ ret = msm_gem_get_and_pin_iova_range(bo->obj, gmu->aspace, &bo->iova,
	1153	+ range_start >> PAGE_SHIFT, range_end >> PAGE_SHIFT);
	1154	+ if (ret) {
	1155	+ drm_gem_object_put(bo->obj);
	1156	+ return ret;
802	1157	}
803	1158
804		- bo->iova = gmu->uncached_iova_base;
	1159	+ bo->virt = msm_gem_get_vaddr(bo->obj);
	1160	+ bo->size = size;
805	1161
806		- for (i = 0; i < count; i++) {
807		- ret = iommu_map(gmu->domain,
808		- bo->iova + (PAGE_SIZE * i),
809		- page_to_phys(bo->pages[i]), PAGE_SIZE,
810		- IOMMU_READ \| IOMMU_WRITE);
811		-
812		- if (ret) {
813		- dev_err(gmu->dev, "Unable to map GMU buffer object\n");
814		-
815		- for (i = i - 1 ; i >= 0; i--)
816		- iommu_unmap(gmu->domain,
817		- bo->iova + (PAGE_SIZE * i),
818		- PAGE_SIZE);
819		-
820		- goto err;
821		- }
822		- }
823		-
824		- bo->virt = vmap(bo->pages, count, VM_IOREMAP,
825		- pgprot_writecombine(PAGE_KERNEL));
826		- if (!bo->virt)
827		- goto err;
828		-
829		- /* Align future IOVA addresses on 1MB boundaries */
830		- gmu->uncached_iova_base += ALIGN(size, SZ_1M);
831		-
832		- return bo;
833		-
834		-err:
835		- for (i = 0; i < count; i++) {
836		- if (bo->pages[i])
837		- __free_pages(bo->pages[i], 0);
838		- }
839		-
840		- kfree(bo->pages);
841		- kfree(bo);
842		-
843		- return ERR_PTR(-ENOMEM);
	1162	+ return 0;
844	1163	}
845	1164
846	1165	static int a6xx_gmu_memory_probe(struct a6xx_gmu *gmu)
847	1166	{
848		- int ret;
	1167	+ struct iommu_domain *domain;
	1168	+ struct msm_mmu *mmu;
849	1169
850		- /*
851		- * The GMU address space is hardcoded to treat the range
852		- * 0x60000000 - 0x80000000 as un-cached memory. All buffers shared
853		- * between the GMU and the CPU will live in this space
854		- */
855		- gmu->uncached_iova_base = 0x60000000;
856		-
857		-
858		- gmu->domain = iommu_domain_alloc(&platform_bus_type);
859		- if (!gmu->domain)
	1170	+ domain = iommu_domain_alloc(&platform_bus_type);
	1171	+ if (!domain)
860	1172	return -ENODEV;
861	1173
862		- ret = iommu_attach_device(gmu->domain, gmu->dev);
863		-
864		- if (ret) {
865		- iommu_domain_free(gmu->domain);
866		- gmu->domain = NULL;
	1174	+ mmu = msm_iommu_new(gmu->dev, domain);
	1175	+ gmu->aspace = msm_gem_address_space_create(mmu, "gmu", 0x0, 0x80000000);
	1176	+ if (IS_ERR(gmu->aspace)) {
	1177	+ iommu_domain_free(domain);
	1178	+ return PTR_ERR(gmu->aspace);
867	1179	}
868	1180
869		- return ret;
870		-}
871		-
872		-/* Get the list of RPMh voltage levels from cmd-db */
873		-static int a6xx_gmu_rpmh_arc_cmds(const char id, void vals, int size)
874		-{
875		- u32 len = cmd_db_read_aux_data_len(id);
876		-
877		- if (!len)
878		- return 0;
879		-
880		- if (WARN_ON(len > size))
881		- return -EINVAL;
882		-
883		- cmd_db_read_aux_data(id, vals, len);
884		-
885		- /*
886		- * The data comes back as an array of unsigned shorts so adjust the
887		- * count accordingly
888		- */
889		- return len >> 1;
	1181	+ return 0;
890	1182	}
891	1183
892	1184	/* Return the 'arc-level' for the given frequency */
893		-static u32 a6xx_gmu_get_arc_level(struct device *dev, unsigned long freq)
	1185	+static unsigned int a6xx_gmu_get_arc_level(struct device *dev,
	1186	+ unsigned long freq)
894	1187	{
895	1188	struct dev_pm_opp *opp;
896		- struct device_node *np;
897		- u32 val = 0;
	1189	+ unsigned int val;
898	1190
899	1191	if (!freq)
900	1192	return 0;
901	1193
902		- opp = dev_pm_opp_find_freq_exact(dev, freq, true);
	1194	+ opp = dev_pm_opp_find_freq_exact(dev, freq, true);
903	1195	if (IS_ERR(opp))
904	1196	return 0;
905	1197
906		- np = dev_pm_opp_get_of_node(opp);
907		-
908		- if (np) {
909		- of_property_read_u32(np, "opp-level", &val);
910		- of_node_put(np);
911		- }
	1198	+ val = dev_pm_opp_get_level(opp);
912	1199
913	1200	dev_pm_opp_put(opp);
914	1201
..	..	@@ -916,16 +1203,35 @@
916	1203	}
917	1204
918	1205	static int a6xx_gmu_rpmh_arc_votes_init(struct device dev, u32 votes,
919		- unsigned long *freqs, int freqs_count,
920		- u16 *pri, int pri_count,
921		- u16 *sec, int sec_count)
	1206	+ unsigned long freqs, int freqs_count, const char id)
922	1207	{
923	1208	int i, j;
	1209	+ const u16 pri, sec;
	1210	+ size_t pri_count, sec_count;
	1211	+
	1212	+ pri = cmd_db_read_aux_data(id, &pri_count);
	1213	+ if (IS_ERR(pri))
	1214	+ return PTR_ERR(pri);
	1215	+ /*
	1216	+ * The data comes back as an array of unsigned shorts so adjust the
	1217	+ * count accordingly
	1218	+ */
	1219	+ pri_count >>= 1;
	1220	+ if (!pri_count)
	1221	+ return -EINVAL;
	1222	+
	1223	+ sec = cmd_db_read_aux_data("mx.lvl", &sec_count);
	1224	+ if (IS_ERR(sec))
	1225	+ return PTR_ERR(sec);
	1226	+
	1227	+ sec_count >>= 1;
	1228	+ if (!sec_count)
	1229	+ return -EINVAL;
924	1230
925	1231	/* Construct a vote for each frequency */
926	1232	for (i = 0; i < freqs_count; i++) {
927	1233	u8 pindex = 0, sindex = 0;
928		- u32 level = a6xx_gmu_get_arc_level(dev, freqs[i]);
	1234	+ unsigned int level = a6xx_gmu_get_arc_level(dev, freqs[i]);
929	1235
930	1236	/* Get the primary index that matches the arc level */
931	1237	for (j = 0; j < pri_count; j++) {
..	..	@@ -936,12 +1242,12 @@
936	1242	}
937	1243
938	1244	if (j == pri_count) {
939		- dev_err(dev,
940		- "Level %u not found in in the RPMh list\n",
941		- level);
942		- dev_err(dev, "Available levels:\n");
	1245	+ DRM_DEV_ERROR(dev,
	1246	+ "Level %u not found in the RPMh list\n",
	1247	+ level);
	1248	+ DRM_DEV_ERROR(dev, "Available levels:\n");
943	1249	for (j = 0; j < pri_count; j++)
944		- dev_err(dev, " %u\n", pri[j]);
	1250	+ DRM_DEV_ERROR(dev, " %u\n", pri[j]);
945	1251
946	1252	return -EINVAL;
947	1253	}
..	..	@@ -979,25 +1285,15 @@
979	1285	struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
980	1286	struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
981	1287	struct msm_gpu *gpu = &adreno_gpu->base;
982		-
983		- u16 gx[16], cx[16], mx[16];
984		- u32 gxcount, cxcount, mxcount;
985	1288	int ret;
986		-
987		- /* Get the list of available voltage levels for each component */
988		- gxcount = a6xx_gmu_rpmh_arc_cmds("gfx.lvl", gx, sizeof(gx));
989		- cxcount = a6xx_gmu_rpmh_arc_cmds("cx.lvl", cx, sizeof(cx));
990		- mxcount = a6xx_gmu_rpmh_arc_cmds("mx.lvl", mx, sizeof(mx));
991	1289
992	1290	/* Build the GX votes */
993	1291	ret = a6xx_gmu_rpmh_arc_votes_init(&gpu->pdev->dev, gmu->gx_arc_votes,
994		- gmu->gpu_freqs, gmu->nr_gpu_freqs,
995		- gx, gxcount, mx, mxcount);
	1292	+ gmu->gpu_freqs, gmu->nr_gpu_freqs, "gfx.lvl");
996	1293
997	1294	/* Build the CX votes */
998	1295	ret \|= a6xx_gmu_rpmh_arc_votes_init(gmu->dev, gmu->cx_arc_votes,
999		- gmu->gmu_freqs, gmu->nr_gmu_freqs,
1000		- cx, cxcount, mx, mxcount);
	1296	+ gmu->gmu_freqs, gmu->nr_gmu_freqs, "cx.lvl");
1001	1297
1002	1298	return ret;
1003	1299	}
..	..	@@ -1048,7 +1344,7 @@
1048	1344	*/
1049	1345	ret = dev_pm_opp_of_add_table(gmu->dev);
1050	1346	if (ret) {
1051		- dev_err(gmu->dev, "Unable to set the OPP table for the GMU\n");
	1347	+ DRM_DEV_ERROR(gmu->dev, "Unable to set the OPP table for the GMU\n");
1052	1348	return ret;
1053	1349	}
1054	1350
..	..	@@ -1062,13 +1358,15 @@
1062	1358	gmu->nr_gpu_freqs = a6xx_gmu_build_freq_table(&gpu->pdev->dev,
1063	1359	gmu->gpu_freqs, ARRAY_SIZE(gmu->gpu_freqs));
1064	1360
	1361	+ gmu->current_perf_index = gmu->nr_gpu_freqs - 1;
	1362	+
1065	1363	/* Build the list of RPMh votes that we'll send to the GMU */
1066	1364	return a6xx_gmu_rpmh_votes_init(gmu);
1067	1365	}
1068	1366
1069	1367	static int a6xx_gmu_clocks_probe(struct a6xx_gmu *gmu)
1070	1368	{
1071		- int ret = msm_clk_bulk_get(gmu->dev, &gmu->clocks);
	1369	+ int ret = devm_clk_bulk_get_all(gmu->dev, &gmu->clocks);
1072	1370
1073	1371	if (ret < 1)
1074	1372	return ret;
..	..	@@ -1089,13 +1387,13 @@
1089	1387	IORESOURCE_MEM, name);
1090	1388
1091	1389	if (!res) {
1092		- dev_err(&pdev->dev, "Unable to find the %s registers\n", name);
	1390	+ DRM_DEV_ERROR(&pdev->dev, "Unable to find the %s registers\n", name);
1093	1391	return ERR_PTR(-EINVAL);
1094	1392	}
1095	1393
1096		- ret = devm_ioremap(&pdev->dev, res->start, resource_size(res));
	1394	+ ret = ioremap(res->start, resource_size(res));
1097	1395	if (!ret) {
1098		- dev_err(&pdev->dev, "Unable to map the %s registers\n", name);
	1396	+ DRM_DEV_ERROR(&pdev->dev, "Unable to map the %s registers\n", name);
1099	1397	return ERR_PTR(-EINVAL);
1100	1398	}
1101	1399
..	..	@@ -1109,10 +1407,10 @@
1109	1407
1110	1408	irq = platform_get_irq_byname(pdev, name);
1111	1409
1112		- ret = devm_request_irq(&pdev->dev, irq, handler, IRQF_TRIGGER_HIGH,
1113		- name, gmu);
	1410	+ ret = request_irq(irq, handler, IRQF_TRIGGER_HIGH, name, gmu);
1114	1411	if (ret) {
1115		- dev_err(&pdev->dev, "Unable to get interrupt %s\n", name);
	1412	+ DRM_DEV_ERROR(&pdev->dev, "Unable to get interrupt %s %d\n",
	1413	+ name, ret);
1116	1414	return ret;
1117	1415	}
1118	1416
..	..	@@ -1124,23 +1422,38 @@
1124	1422	void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu)
1125	1423	{
1126	1424	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
	1425	+ struct platform_device *pdev = to_platform_device(gmu->dev);
1127	1426
1128		- if (IS_ERR_OR_NULL(gmu->mmio))
	1427	+ if (!gmu->initialized)
1129	1428	return;
1130	1429
1131		- pm_runtime_disable(gmu->dev);
1132		- a6xx_gmu_stop(a6xx_gpu);
	1430	+ pm_runtime_force_suspend(gmu->dev);
1133	1431
1134		- a6xx_gmu_irq_disable(gmu);
1135		- a6xx_gmu_memory_free(gmu, gmu->hfi);
	1432	+ if (!IS_ERR_OR_NULL(gmu->gxpd)) {
	1433	+ pm_runtime_disable(gmu->gxpd);
	1434	+ dev_pm_domain_detach(gmu->gxpd, false);
	1435	+ }
1136	1436
1137		- iommu_detach_device(gmu->domain, gmu->dev);
	1437	+ iounmap(gmu->mmio);
	1438	+ if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "rscc"))
	1439	+ iounmap(gmu->rscc);
	1440	+ gmu->mmio = NULL;
	1441	+ gmu->rscc = NULL;
1138	1442
1139		- iommu_domain_free(gmu->domain);
	1443	+ a6xx_gmu_memory_free(gmu);
	1444	+
	1445	+ free_irq(gmu->gmu_irq, gmu);
	1446	+ free_irq(gmu->hfi_irq, gmu);
	1447	+
	1448	+ /* Drop reference taken in of_find_device_by_node */
	1449	+ put_device(gmu->dev);
	1450	+
	1451	+ gmu->initialized = false;
1140	1452	}
1141	1453
1142		-int a6xx_gmu_probe(struct a6xx_gpu a6xx_gpu, struct device_node node)
	1454	+int a6xx_gmu_init(struct a6xx_gpu a6xx_gpu, struct device_node node)
1143	1455	{
	1456	+ struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
1144	1457	struct a6xx_gmu *gmu = &a6xx_gpu->gmu;
1145	1458	struct platform_device *pdev = of_find_device_by_node(node);
1146	1459	int ret;
..	..	@@ -1156,46 +1469,83 @@
1156	1469	gmu->idle_level = GMU_IDLE_STATE_ACTIVE;
1157	1470
1158	1471	pm_runtime_enable(gmu->dev);
1159		- gmu->gx = devm_regulator_get(gmu->dev, "vdd");
1160	1472
1161	1473	/* Get the list of clocks */
1162	1474	ret = a6xx_gmu_clocks_probe(gmu);
1163	1475	if (ret)
1164		- return ret;
	1476	+ goto err_put_device;
1165	1477
1166		- /* Set up the IOMMU context bank */
1167	1478	ret = a6xx_gmu_memory_probe(gmu);
1168	1479	if (ret)
1169		- return ret;
	1480	+ goto err_put_device;
	1481	+
	1482	+ /* Allocate memory for the GMU dummy page */
	1483	+ ret = a6xx_gmu_memory_alloc(gmu, &gmu->dummy, SZ_4K, 0x60000000);
	1484	+ if (ret)
	1485	+ goto err_memory;
	1486	+
	1487	+ if (adreno_is_a650(adreno_gpu)) {
	1488	+ ret = a6xx_gmu_memory_alloc(gmu, &gmu->icache,
	1489	+ SZ_16M - SZ_16K, 0x04000);
	1490	+ if (ret)
	1491	+ goto err_memory;
	1492	+ } else if (adreno_is_a640(adreno_gpu)) {
	1493	+ ret = a6xx_gmu_memory_alloc(gmu, &gmu->icache,
	1494	+ SZ_256K - SZ_16K, 0x04000);
	1495	+ if (ret)
	1496	+ goto err_memory;
	1497	+
	1498	+ ret = a6xx_gmu_memory_alloc(gmu, &gmu->dcache,
	1499	+ SZ_256K - SZ_16K, 0x44000);
	1500	+ if (ret)
	1501	+ goto err_memory;
	1502	+ } else {
	1503	+ /* HFI v1, has sptprac */
	1504	+ gmu->legacy = true;
	1505	+
	1506	+ /* Allocate memory for the GMU debug region */
	1507	+ ret = a6xx_gmu_memory_alloc(gmu, &gmu->debug, SZ_16K, 0);
	1508	+ if (ret)
	1509	+ goto err_memory;
	1510	+ }
1170	1511
1171	1512	/* Allocate memory for for the HFI queues */
1172		- gmu->hfi = a6xx_gmu_memory_alloc(gmu, SZ_16K);
1173		- if (IS_ERR(gmu->hfi))
1174		- goto err;
	1513	+ ret = a6xx_gmu_memory_alloc(gmu, &gmu->hfi, SZ_16K, 0);
	1514	+ if (ret)
	1515	+ goto err_memory;
1175	1516
1176		- /* Allocate memory for the GMU debug region */
1177		- gmu->debug = a6xx_gmu_memory_alloc(gmu, SZ_16K);
1178		- if (IS_ERR(gmu->debug))
1179		- goto err;
	1517	+ /* Allocate memory for the GMU log region */
	1518	+ ret = a6xx_gmu_memory_alloc(gmu, &gmu->log, SZ_4K, 0);
	1519	+ if (ret)
	1520	+ goto err_memory;
1180	1521
1181	1522	/* Map the GMU registers */
1182	1523	gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu");
	1524	+ if (IS_ERR(gmu->mmio)) {
	1525	+ ret = PTR_ERR(gmu->mmio);
	1526	+ goto err_memory;
	1527	+ }
1183	1528
1184		- /* Map the GPU power domain controller registers */
1185		- gmu->pdc_mmio = a6xx_gmu_get_mmio(pdev, "gmu_pdc");
1186		-
1187		- if (IS_ERR(gmu->mmio) \|\| IS_ERR(gmu->pdc_mmio))
1188		- goto err;
	1529	+ if (adreno_is_a650(adreno_gpu)) {
	1530	+ gmu->rscc = a6xx_gmu_get_mmio(pdev, "rscc");
	1531	+ if (IS_ERR(gmu->rscc))
	1532	+ goto err_mmio;
	1533	+ } else {
	1534	+ gmu->rscc = gmu->mmio + 0x23000;
	1535	+ }
1189	1536
1190	1537	/* Get the HFI and GMU interrupts */
1191	1538	gmu->hfi_irq = a6xx_gmu_get_irq(gmu, pdev, "hfi", a6xx_hfi_irq);
1192	1539	gmu->gmu_irq = a6xx_gmu_get_irq(gmu, pdev, "gmu", a6xx_gmu_irq);
1193	1540
1194	1541	if (gmu->hfi_irq < 0 \|\| gmu->gmu_irq < 0)
1195		- goto err;
	1542	+ goto err_mmio;
1196	1543
1197		- /* Set up a tasklet to handle GMU HFI responses */
1198		- tasklet_init(&gmu->hfi_tasklet, a6xx_hfi_task, (unsigned long) gmu);
	1544	+ /*
	1545	+ * Get a link to the GX power domain to reset the GPU in case of GMU
	1546	+ * crash
	1547	+ */
	1548	+ gmu->gxpd = dev_pm_domain_attach_by_name(gmu->dev, "gx");
1199	1549
1200	1550	/* Get the power levels for the GMU and GPU */
1201	1551	a6xx_gmu_pwrlevels_probe(gmu);
..	..	@@ -1203,15 +1553,24 @@
1203	1553	/* Set up the HFI queues */
1204	1554	a6xx_hfi_init(gmu);
1205	1555
	1556	+ gmu->initialized = true;
	1557	+
1206	1558	return 0;
1207		-err:
1208		- a6xx_gmu_memory_free(gmu, gmu->hfi);
1209	1559
1210		- if (gmu->domain) {
1211		- iommu_detach_device(gmu->domain, gmu->dev);
	1560	+err_mmio:
	1561	+ iounmap(gmu->mmio);
	1562	+ if (platform_get_resource_byname(pdev, IORESOURCE_MEM, "rscc"))
	1563	+ iounmap(gmu->rscc);
	1564	+ free_irq(gmu->gmu_irq, gmu);
	1565	+ free_irq(gmu->hfi_irq, gmu);
1212	1566
1213		- iommu_domain_free(gmu->domain);
1214		- }
	1567	+ ret = -ENODEV;
1215	1568
1216		- return -ENODEV;
	1569	+err_memory:
	1570	+ a6xx_gmu_memory_free(gmu);
	1571	+err_put_device:
	1572	+ /* Drop reference taken in of_find_device_by_node */
	1573	+ put_device(gmu->dev);
	1574	+
	1575	+ return ret;
1217	1576	}