forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d 
 kernel/drivers/gpu/drm/amd/amdgpu/vega10_ih.c
....@@ -20,7 +20,9 @@
2020  * OTHER DEALINGS IN THE SOFTWARE.
2121  *
2222  */
23
-#include <drm/drmP.h>
23
+
24
+#include <linux/pci.h>
25
+
2426 #include "amdgpu.h"
2527 #include "amdgpu_ih.h"
2628 #include "soc15.h"
....@@ -31,7 +33,7 @@
3133 #include "soc15_common.h"
3234 #include "vega10_ih.h"
3335 
34
-
36
+#define MAX_REARM_RETRY 10
3537 
3638 static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev);
3739 
....@@ -48,8 +50,47 @@
4850 
4951 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 1);
5052 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 1);
51
-	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
53
+	if (amdgpu_sriov_vf(adev)) {
54
+		if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) {
55
+			DRM_ERROR("PSP program IH_RB_CNTL failed!\n");
56
+			return;
57
+		}
58
+	} else {
59
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
60
+	}
5261 	adev->irq.ih.enabled = true;
62
+
63
+	if (adev->irq.ih1.ring_size) {
64
+		ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1);
65
+		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1,
66
+					   RB_ENABLE, 1);
67
+		if (amdgpu_sriov_vf(adev)) {
68
+			if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1,
69
+						ih_rb_cntl)) {
70
+				DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n");
71
+				return;
72
+			}
73
+		} else {
74
+			WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
75
+		}
76
+		adev->irq.ih1.enabled = true;
77
+	}
78
+
79
+	if (adev->irq.ih2.ring_size) {
80
+		ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2);
81
+		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2,
82
+					   RB_ENABLE, 1);
83
+		if (amdgpu_sriov_vf(adev)) {
84
+			if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2,
85
+						ih_rb_cntl)) {
86
+				DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n");
87
+				return;
88
+			}
89
+		} else {
90
+			WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
91
+		}
92
+		adev->irq.ih2.enabled = true;
93
+	}
5394 }
5495 
5596 /**
....@@ -65,12 +106,103 @@
65106 
66107 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_ENABLE, 0);
67108 	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, ENABLE_INTR, 0);
68
-	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
109
+	if (amdgpu_sriov_vf(adev)) {
110
+		if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) {
111
+			DRM_ERROR("PSP program IH_RB_CNTL failed!\n");
112
+			return;
113
+		}
114
+	} else {
115
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
116
+	}
117
+
69118 	/* set rptr, wptr to 0 */
70119 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
71120 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
72121 	adev->irq.ih.enabled = false;
73122 	adev->irq.ih.rptr = 0;
123
+
124
+	if (adev->irq.ih1.ring_size) {
125
+		ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1);
126
+		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING1,
127
+					   RB_ENABLE, 0);
128
+		if (amdgpu_sriov_vf(adev)) {
129
+			if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1,
130
+						ih_rb_cntl)) {
131
+				DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n");
132
+				return;
133
+			}
134
+		} else {
135
+			WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
136
+		}
137
+		/* set rptr, wptr to 0 */
138
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0);
139
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0);
140
+		adev->irq.ih1.enabled = false;
141
+		adev->irq.ih1.rptr = 0;
142
+	}
143
+
144
+	if (adev->irq.ih2.ring_size) {
145
+		ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2);
146
+		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL_RING2,
147
+					   RB_ENABLE, 0);
148
+		if (amdgpu_sriov_vf(adev)) {
149
+			if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2,
150
+						ih_rb_cntl)) {
151
+				DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n");
152
+				return;
153
+			}
154
+		} else {
155
+			WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
156
+		}
157
+
158
+		/* set rptr, wptr to 0 */
159
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0);
160
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0);
161
+		adev->irq.ih2.enabled = false;
162
+		adev->irq.ih2.rptr = 0;
163
+	}
164
+}
165
+
166
+static uint32_t vega10_ih_rb_cntl(struct amdgpu_ih_ring *ih, uint32_t ih_rb_cntl)
167
+{
168
+	int rb_bufsz = order_base_2(ih->ring_size / 4);
169
+
170
+	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
171
+				   MC_SPACE, ih->use_bus_addr ? 1 : 4);
172
+	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
173
+				   WPTR_OVERFLOW_CLEAR, 1);
174
+	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
175
+				   WPTR_OVERFLOW_ENABLE, 1);
176
+	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
177
+	/* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register
178
+	 * value is written to memory
179
+	 */
180
+	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
181
+				   WPTR_WRITEBACK_ENABLE, 1);
182
+	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
183
+	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
184
+	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);
185
+
186
+	return ih_rb_cntl;
187
+}
188
+
189
+static uint32_t vega10_ih_doorbell_rptr(struct amdgpu_ih_ring *ih)
190
+{
191
+	u32 ih_doorbell_rtpr = 0;
192
+
193
+	if (ih->use_doorbell) {
194
+		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
195
+						 IH_DOORBELL_RPTR, OFFSET,
196
+						 ih->doorbell_index);
197
+		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
198
+						 IH_DOORBELL_RPTR,
199
+						 ENABLE, 1);
200
+	} else {
201
+		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr,
202
+						 IH_DOORBELL_RPTR,
203
+						 ENABLE, 0);
204
+	}
205
+	return ih_doorbell_rtpr;
74206 }
75207 
76208 /**
....@@ -86,68 +218,117 @@
86218  */
87219 static int vega10_ih_irq_init(struct amdgpu_device *adev)
88220 {
221
+	struct amdgpu_ih_ring *ih;
222
+	u32 ih_rb_cntl, ih_chicken;
89223 	int ret = 0;
90
-	int rb_bufsz;
91
-	u32 ih_rb_cntl, ih_doorbell_rtpr;
92224 	u32 tmp;
93
-	u64 wptr_off;
94225 
95226 	/* disable irqs */
96227 	vega10_ih_disable_interrupts(adev);
97228 
98
-	adev->nbio_funcs->ih_control(adev);
229
+	adev->nbio.funcs->ih_control(adev);
230
+
231
+	ih = &adev->irq.ih;
232
+	/* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
233
+	WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, ih->gpu_addr >> 8);
234
+	WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (ih->gpu_addr >> 40) & 0xff);
99235 
100236 	ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL);
101
-	/* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/
102
-	if (adev->irq.ih.use_bus_addr) {
103
-		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.rb_dma_addr >> 8);
104
-		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, ((u64)adev->irq.ih.rb_dma_addr >> 40) & 0xff);
105
-		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 1);
237
+	ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl);
238
+	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM,
239
+				   !!adev->irq.msi_enabled);
240
+	if (amdgpu_sriov_vf(adev)) {
241
+		if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL, ih_rb_cntl)) {
242
+			DRM_ERROR("PSP program IH_RB_CNTL failed!\n");
243
+			return -ETIMEDOUT;
244
+		}
106245 	} else {
107
-		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8);
108
-		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI, (adev->irq.ih.gpu_addr >> 40) & 0xff);
109
-		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SPACE, 4);
246
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
110247 	}
111
-	rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4);
112
-	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
113
-	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_OVERFLOW_ENABLE, 1);
114
-	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RB_SIZE, rb_bufsz);
115
-	/* Ring Buffer write pointer writeback. If enabled, IH_RB_WPTR register value is written to memory */
116
-	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, WPTR_WRITEBACK_ENABLE, 1);
117
-	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_SNOOP, 1);
118
-	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_RO, 0);
119
-	ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, MC_VMID, 0);
120248 
121
-	if (adev->irq.msi_enabled)
122
-		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL, RPTR_REARM, 1);
123
-
124
-	WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL, ih_rb_cntl);
249
+	if ((adev->asic_type == CHIP_ARCTURUS &&
250
+	     adev->firmware.load_type == AMDGPU_FW_LOAD_DIRECT) ||
251
+	    adev->asic_type == CHIP_RENOIR) {
252
+		ih_chicken = RREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN);
253
+		if (adev->irq.ih.use_bus_addr) {
254
+			ih_chicken = REG_SET_FIELD(ih_chicken, IH_CHICKEN,
255
+						   MC_SPACE_GPA_ENABLE, 1);
256
+		} else {
257
+			ih_chicken = REG_SET_FIELD(ih_chicken, IH_CHICKEN,
258
+						   MC_SPACE_FBPA_ENABLE, 1);
259
+		}
260
+		WREG32_SOC15(OSSSYS, 0, mmIH_CHICKEN, ih_chicken);
261
+	}
125262 
126263 	/* set the writeback address whether it's enabled or not */
127
-	if (adev->irq.ih.use_bus_addr)
128
-		wptr_off = adev->irq.ih.rb_dma_addr + (adev->irq.ih.wptr_offs * 4);
129
-	else
130
-		wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4);
131
-	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO, lower_32_bits(wptr_off));
132
-	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI, upper_32_bits(wptr_off) & 0xFFFF);
264
+	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_LO,
265
+		     lower_32_bits(ih->wptr_addr));
266
+	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_ADDR_HI,
267
+		     upper_32_bits(ih->wptr_addr) & 0xFFFF);
133268 
134269 	/* set rptr, wptr to 0 */
135
-	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
136270 	WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR, 0);
271
+	WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, 0);
137272 
138
-	ih_doorbell_rtpr = RREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR);
139
-	if (adev->irq.ih.use_doorbell) {
140
-		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
141
-						 OFFSET, adev->irq.ih.doorbell_index);
142
-		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
143
-						 ENABLE, 1);
144
-	} else {
145
-		ih_doorbell_rtpr = REG_SET_FIELD(ih_doorbell_rtpr, IH_DOORBELL_RPTR,
146
-						 ENABLE, 0);
273
+	WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR,
274
+		     vega10_ih_doorbell_rptr(ih));
275
+
276
+	ih = &adev->irq.ih1;
277
+	if (ih->ring_size) {
278
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING1, ih->gpu_addr >> 8);
279
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING1,
280
+			     (ih->gpu_addr >> 40) & 0xff);
281
+
282
+		ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1);
283
+		ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl);
284
+		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
285
+					   WPTR_OVERFLOW_ENABLE, 0);
286
+		ih_rb_cntl = REG_SET_FIELD(ih_rb_cntl, IH_RB_CNTL,
287
+					   RB_FULL_DRAIN_ENABLE, 1);
288
+		if (amdgpu_sriov_vf(adev)) {
289
+			if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING1,
290
+						ih_rb_cntl)) {
291
+				DRM_ERROR("program IH_RB_CNTL_RING1 failed!\n");
292
+				return -ETIMEDOUT;
293
+			}
294
+		} else {
295
+			WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING1, ih_rb_cntl);
296
+		}
297
+
298
+		/* set rptr, wptr to 0 */
299
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING1, 0);
300
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, 0);
301
+
302
+		WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING1,
303
+			     vega10_ih_doorbell_rptr(ih));
147304 	}
148
-	WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR, ih_doorbell_rtpr);
149
-	adev->nbio_funcs->ih_doorbell_range(adev, adev->irq.ih.use_doorbell,
150
-					    adev->irq.ih.doorbell_index);
305
+
306
+	ih = &adev->irq.ih2;
307
+	if (ih->ring_size) {
308
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_RING2, ih->gpu_addr >> 8);
309
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_BASE_HI_RING2,
310
+			     (ih->gpu_addr >> 40) & 0xff);
311
+
312
+		ih_rb_cntl = RREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2);
313
+		ih_rb_cntl = vega10_ih_rb_cntl(ih, ih_rb_cntl);
314
+
315
+		if (amdgpu_sriov_vf(adev)) {
316
+			if (psp_reg_program(&adev->psp, PSP_REG_IH_RB_CNTL_RING2,
317
+						ih_rb_cntl)) {
318
+				DRM_ERROR("program IH_RB_CNTL_RING2 failed!\n");
319
+				return -ETIMEDOUT;
320
+			}
321
+		} else {
322
+			WREG32_SOC15(OSSSYS, 0, mmIH_RB_CNTL_RING2, ih_rb_cntl);
323
+		}
324
+
325
+		/* set rptr, wptr to 0 */
326
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_WPTR_RING2, 0);
327
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, 0);
328
+
329
+		WREG32_SOC15(OSSSYS, 0, mmIH_DOORBELL_RPTR_RING2,
330
+			     vega10_ih_doorbell_rptr(ih));
331
+	}
151332 
152333 	tmp = RREG32_SOC15(OSSSYS, 0, mmIH_STORM_CLIENT_LIST_CNTL);
153334 	tmp = REG_SET_FIELD(tmp, IH_STORM_CLIENT_LIST_CNTL,
....@@ -191,115 +372,58 @@
191372  * ring buffer overflow and deal with it.
192373  * Returns the value of the wptr.
193374  */
194
-static u32 vega10_ih_get_wptr(struct amdgpu_device *adev)
375
+static u32 vega10_ih_get_wptr(struct amdgpu_device *adev,
376
+			      struct amdgpu_ih_ring *ih)
195377 {
196
-	u32 wptr, tmp;
378
+	u32 wptr, reg, tmp;
197379 
198
-	if (adev->irq.ih.use_bus_addr)
199
-		wptr = le32_to_cpu(adev->irq.ih.ring[adev->irq.ih.wptr_offs]);
380
+	wptr = le32_to_cpu(*ih->wptr_cpu);
381
+
382
+	if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
383
+		goto out;
384
+
385
+	/* Double check that the overflow wasn't already cleared. */
386
+
387
+	if (ih == &adev->irq.ih)
388
+		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR);
389
+	else if (ih == &adev->irq.ih1)
390
+		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING1);
391
+	else if (ih == &adev->irq.ih2)
392
+		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_WPTR_RING2);
200393 	else
201
-		wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]);
394
+		BUG();
202395 
203
-	if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) {
204
-		wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
396
+	wptr = RREG32_NO_KIQ(reg);
397
+	if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW))
398
+		goto out;
205399 
206
-		/* When a ring buffer overflow happen start parsing interrupt
207
-		 * from the last not overwritten vector (wptr + 32). Hopefully
208
-		 * this should allow us to catchup.
209
-		 */
210
-		tmp = (wptr + 32) & adev->irq.ih.ptr_mask;
211
-		dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
212
-			wptr, adev->irq.ih.rptr, tmp);
213
-		adev->irq.ih.rptr = tmp;
400
+	wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0);
214401 
215
-		tmp = RREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL));
216
-		tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
217
-		WREG32_NO_KIQ(SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL), tmp);
218
-	}
219
-	return (wptr & adev->irq.ih.ptr_mask);
220
-}
221
-
222
-/**
223
- * vega10_ih_prescreen_iv - prescreen an interrupt vector
224
- *
225
- * @adev: amdgpu_device pointer
226
- *
227
- * Returns true if the interrupt vector should be further processed.
228
- */
229
-static bool vega10_ih_prescreen_iv(struct amdgpu_device *adev)
230
-{
231
-	u32 ring_index = adev->irq.ih.rptr >> 2;
232
-	u32 dw0, dw3, dw4, dw5;
233
-	u16 pasid;
234
-	u64 addr, key;
235
-	struct amdgpu_vm *vm;
236
-	int r;
237
-
238
-	dw0 = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
239
-	dw3 = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
240
-	dw4 = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
241
-	dw5 = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);
242
-
243
-	/* Filter retry page faults, let only the first one pass. If
244
-	 * there are too many outstanding faults, ignore them until
245
-	 * some faults get cleared.
402
+	/* When a ring buffer overflow happen start parsing interrupt
403
+	 * from the last not overwritten vector (wptr + 32). Hopefully
404
+	 * this should allow us to catchup.
246405 	 */
247
-	switch (dw0 & 0xff) {
248
-	case SOC15_IH_CLIENTID_VMC:
249
-	case SOC15_IH_CLIENTID_UTCL2:
250
-		break;
251
-	default:
252
-		/* Not a VM fault */
253
-		return true;
254
-	}
406
+	tmp = (wptr + 32) & ih->ptr_mask;
407
+	dev_warn(adev->dev, "IH ring buffer overflow "
408
+		 "(0x%08X, 0x%08X, 0x%08X)\n",
409
+		 wptr, ih->rptr, tmp);
410
+	ih->rptr = tmp;
255411 
256
-	pasid = dw3 & 0xffff;
257
-	/* No PASID, can't identify faulting process */
258
-	if (!pasid)
259
-		return true;
412
+	if (ih == &adev->irq.ih)
413
+		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL);
414
+	else if (ih == &adev->irq.ih1)
415
+		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING1);
416
+	else if (ih == &adev->irq.ih2)
417
+		reg = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_CNTL_RING2);
418
+	else
419
+		BUG();
260420 
261
-	/* Not a retry fault, check fault credit */
262
-	if (!(dw5 & 0x80)) {
263
-		if (!amdgpu_vm_pasid_fault_credit(adev, pasid))
264
-			goto ignore_iv;
265
-		return true;
266
-	}
421
+	tmp = RREG32_NO_KIQ(reg);
422
+	tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1);
423
+	WREG32_NO_KIQ(reg, tmp);
267424 
268
-	addr = ((u64)(dw5 & 0xf) << 44) | ((u64)dw4 << 12);
269
-	key = AMDGPU_VM_FAULT(pasid, addr);
270
-	r = amdgpu_ih_add_fault(adev, key);
271
-
272
-	/* Hash table is full or the fault is already being processed,
273
-	 * ignore further page faults
274
-	 */
275
-	if (r != 0)
276
-		goto ignore_iv;
277
-
278
-	/* Track retry faults in per-VM fault FIFO. */
279
-	spin_lock(&adev->vm_manager.pasid_lock);
280
-	vm = idr_find(&adev->vm_manager.pasid_idr, pasid);
281
-	if (!vm) {
282
-		/* VM not found, process it normally */
283
-		spin_unlock(&adev->vm_manager.pasid_lock);
284
-		amdgpu_ih_clear_fault(adev, key);
285
-		return true;
286
-	}
287
-	/* No locking required with single writer and single reader */
288
-	r = kfifo_put(&vm->faults, key);
289
-	if (!r) {
290
-		/* FIFO is full. Ignore it until there is space */
291
-		spin_unlock(&adev->vm_manager.pasid_lock);
292
-		amdgpu_ih_clear_fault(adev, key);
293
-		goto ignore_iv;
294
-	}
295
-	spin_unlock(&adev->vm_manager.pasid_lock);
296
-
297
-	/* It's the first fault for this address, process it normally */
298
-	return true;
299
-
300
-ignore_iv:
301
-	adev->irq.ih.rptr += 32;
302
-	return false;
425
+out:
426
+	return (wptr & ih->ptr_mask);
303427 }
304428 
305429 /**
....@@ -311,20 +435,21 @@
311435  * position and also advance the position.
312436  */
313437 static void vega10_ih_decode_iv(struct amdgpu_device *adev,
314
-				 struct amdgpu_iv_entry *entry)
438
+				struct amdgpu_ih_ring *ih,
439
+				struct amdgpu_iv_entry *entry)
315440 {
316441 	/* wptr/rptr are in bytes! */
317
-	u32 ring_index = adev->irq.ih.rptr >> 2;
442
+	u32 ring_index = ih->rptr >> 2;
318443 	uint32_t dw[8];
319444 
320
-	dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]);
321
-	dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]);
322
-	dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]);
323
-	dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]);
324
-	dw[4] = le32_to_cpu(adev->irq.ih.ring[ring_index + 4]);
325
-	dw[5] = le32_to_cpu(adev->irq.ih.ring[ring_index + 5]);
326
-	dw[6] = le32_to_cpu(adev->irq.ih.ring[ring_index + 6]);
327
-	dw[7] = le32_to_cpu(adev->irq.ih.ring[ring_index + 7]);
445
+	dw[0] = le32_to_cpu(ih->ring[ring_index + 0]);
446
+	dw[1] = le32_to_cpu(ih->ring[ring_index + 1]);
447
+	dw[2] = le32_to_cpu(ih->ring[ring_index + 2]);
448
+	dw[3] = le32_to_cpu(ih->ring[ring_index + 3]);
449
+	dw[4] = le32_to_cpu(ih->ring[ring_index + 4]);
450
+	dw[5] = le32_to_cpu(ih->ring[ring_index + 5]);
451
+	dw[6] = le32_to_cpu(ih->ring[ring_index + 6]);
452
+	dw[7] = le32_to_cpu(ih->ring[ring_index + 7]);
328453 
329454 	entry->client_id = dw[0] & 0xff;
330455 	entry->src_id = (dw[0] >> 8) & 0xff;
....@@ -340,9 +465,40 @@
340465 	entry->src_data[2] = dw[6];
341466 	entry->src_data[3] = dw[7];
342467 
343
-
344468 	/* wptr/rptr are in bytes! */
345
-	adev->irq.ih.rptr += 32;
469
+	ih->rptr += 32;
470
+}
471
+
472
+/**
473
+ * vega10_ih_irq_rearm - rearm IRQ if lost
474
+ *
475
+ * @adev: amdgpu_device pointer
476
+ *
477
+ */
478
+static void vega10_ih_irq_rearm(struct amdgpu_device *adev,
479
+			       struct amdgpu_ih_ring *ih)
480
+{
481
+	uint32_t reg_rptr = 0;
482
+	uint32_t v = 0;
483
+	uint32_t i = 0;
484
+
485
+	if (ih == &adev->irq.ih)
486
+		reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR);
487
+	else if (ih == &adev->irq.ih1)
488
+		reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING1);
489
+	else if (ih == &adev->irq.ih2)
490
+		reg_rptr = SOC15_REG_OFFSET(OSSSYS, 0, mmIH_RB_RPTR_RING2);
491
+	else
492
+		return;
493
+
494
+	/* Rearm IRQ / re-wwrite doorbell if doorbell write is lost */
495
+	for (i = 0; i < MAX_REARM_RETRY; i++) {
496
+		v = RREG32_NO_KIQ(reg_rptr);
497
+		if ((v < ih->ring_size) && (v != ih->rptr))
498
+			WDOORBELL32(ih->doorbell_index, ih->rptr);
499
+		else
500
+			break;
501
+	}
346502 }
347503 
348504 /**
....@@ -352,18 +508,62 @@
352508  *
353509  * Set the IH ring buffer rptr.
354510  */
355
-static void vega10_ih_set_rptr(struct amdgpu_device *adev)
511
+static void vega10_ih_set_rptr(struct amdgpu_device *adev,
512
+			       struct amdgpu_ih_ring *ih)
356513 {
357
-	if (adev->irq.ih.use_doorbell) {
514
+	if (ih->use_doorbell) {
358515 		/* XXX check if swapping is necessary on BE */
359
-		if (adev->irq.ih.use_bus_addr)
360
-			adev->irq.ih.ring[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
361
-		else
362
-			adev->wb.wb[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr;
363
-		WDOORBELL32(adev->irq.ih.doorbell_index, adev->irq.ih.rptr);
364
-	} else {
365
-		WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, adev->irq.ih.rptr);
516
+		*ih->rptr_cpu = ih->rptr;
517
+		WDOORBELL32(ih->doorbell_index, ih->rptr);
518
+
519
+		if (amdgpu_sriov_vf(adev))
520
+			vega10_ih_irq_rearm(adev, ih);
521
+	} else if (ih == &adev->irq.ih) {
522
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR, ih->rptr);
523
+	} else if (ih == &adev->irq.ih1) {
524
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING1, ih->rptr);
525
+	} else if (ih == &adev->irq.ih2) {
526
+		WREG32_SOC15(OSSSYS, 0, mmIH_RB_RPTR_RING2, ih->rptr);
366527 	}
528
+}
529
+
530
+/**
531
+ * vega10_ih_self_irq - dispatch work for ring 1 and 2
532
+ *
533
+ * @adev: amdgpu_device pointer
534
+ * @source: irq source
535
+ * @entry: IV with WPTR update
536
+ *
537
+ * Update the WPTR from the IV and schedule work to handle the entries.
538
+ */
539
+static int vega10_ih_self_irq(struct amdgpu_device *adev,
540
+			      struct amdgpu_irq_src *source,
541
+			      struct amdgpu_iv_entry *entry)
542
+{
543
+	uint32_t wptr = cpu_to_le32(entry->src_data[0]);
544
+
545
+	switch (entry->ring_id) {
546
+	case 1:
547
+		*adev->irq.ih1.wptr_cpu = wptr;
548
+		schedule_work(&adev->irq.ih1_work);
549
+		break;
550
+	case 2:
551
+		*adev->irq.ih2.wptr_cpu = wptr;
552
+		schedule_work(&adev->irq.ih2_work);
553
+		break;
554
+	default: break;
555
+	}
556
+	return 0;
557
+}
558
+
559
+static const struct amdgpu_irq_src_funcs vega10_ih_self_irq_funcs = {
560
+	.process = vega10_ih_self_irq,
561
+};
562
+
563
+static void vega10_ih_set_self_irq_funcs(struct amdgpu_device *adev)
564
+{
565
+	adev->irq.self_irq.num_types = 0;
566
+	adev->irq.self_irq.funcs = &vega10_ih_self_irq_funcs;
367567 }
368568 
369569 static int vega10_ih_early_init(void *handle)
....@@ -371,28 +571,40 @@
371571 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
372572 
373573 	vega10_ih_set_interrupt_funcs(adev);
574
+	vega10_ih_set_self_irq_funcs(adev);
374575 	return 0;
375576 }
376577 
377578 static int vega10_ih_sw_init(void *handle)
378579 {
379
-	int r;
380580 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
581
+	int r;
381582 
382
-	r = amdgpu_ih_ring_init(adev, 256 * 1024, true);
583
+	r = amdgpu_irq_add_id(adev, SOC15_IH_CLIENTID_IH, 0,
584
+			      &adev->irq.self_irq);
585
+	if (r)
586
+		return r;
587
+
588
+	r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 256 * 1024, true);
383589 	if (r)
384590 		return r;
385591 
386592 	adev->irq.ih.use_doorbell = true;
387
-	adev->irq.ih.doorbell_index = AMDGPU_DOORBELL64_IH << 1;
593
+	adev->irq.ih.doorbell_index = adev->doorbell_index.ih << 1;
388594 
389
-	adev->irq.ih.faults = kmalloc(sizeof(*adev->irq.ih.faults), GFP_KERNEL);
390
-	if (!adev->irq.ih.faults)
391
-		return -ENOMEM;
392
-	INIT_CHASH_TABLE(adev->irq.ih.faults->hash,
393
-			 AMDGPU_PAGEFAULT_HASH_BITS, 8, 0);
394
-	spin_lock_init(&adev->irq.ih.faults->lock);
395
-	adev->irq.ih.faults->count = 0;
595
+	r = amdgpu_ih_ring_init(adev, &adev->irq.ih1, PAGE_SIZE, true);
596
+	if (r)
597
+		return r;
598
+
599
+	adev->irq.ih1.use_doorbell = true;
600
+	adev->irq.ih1.doorbell_index = (adev->doorbell_index.ih + 1) << 1;
601
+
602
+	r = amdgpu_ih_ring_init(adev, &adev->irq.ih2, PAGE_SIZE, true);
603
+	if (r)
604
+		return r;
605
+
606
+	adev->irq.ih2.use_doorbell = true;
607
+	adev->irq.ih2.doorbell_index = (adev->doorbell_index.ih + 2) << 1;
396608 
397609 	r = amdgpu_irq_init(adev);
398610 
....@@ -404,10 +616,9 @@
404616 	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
405617 
406618 	amdgpu_irq_fini(adev);
407
-	amdgpu_ih_ring_fini(adev);
408
-
409
-	kfree(adev->irq.ih.faults);
410
-	adev->irq.ih.faults = NULL;
619
+	amdgpu_ih_ring_fini(adev, &adev->irq.ih2);
620
+	amdgpu_ih_ring_fini(adev, &adev->irq.ih1);
621
+	amdgpu_ih_ring_fini(adev, &adev->irq.ih);
411622 
412623 	return 0;
413624 }
....@@ -466,10 +677,49 @@
466677 	return 0;
467678 }
468679 
680
+static void vega10_ih_update_clockgating_state(struct amdgpu_device *adev,
681
+					       bool enable)
682
+{
683
+	uint32_t data, def, field_val;
684
+
685
+	if (adev->cg_flags & AMD_CG_SUPPORT_IH_CG) {
686
+		def = data = RREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL);
687
+		field_val = enable ? 0 : 1;
688
+		/**
689
+		 * Vega10 does not have IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE
690
+		 * and IH_BUFFER_MEM_CLK_SOFT_OVERRIDE field.
691
+		 */
692
+		if (adev->asic_type > CHIP_VEGA10) {
693
+			data = REG_SET_FIELD(data, IH_CLK_CTRL,
694
+				     IH_RETRY_INT_CAM_MEM_CLK_SOFT_OVERRIDE, field_val);
695
+			data = REG_SET_FIELD(data, IH_CLK_CTRL,
696
+				     IH_BUFFER_MEM_CLK_SOFT_OVERRIDE, field_val);
697
+		}
698
+
699
+		data = REG_SET_FIELD(data, IH_CLK_CTRL,
700
+				     DBUS_MUX_CLK_SOFT_OVERRIDE, field_val);
701
+		data = REG_SET_FIELD(data, IH_CLK_CTRL,
702
+				     OSSSYS_SHARE_CLK_SOFT_OVERRIDE, field_val);
703
+		data = REG_SET_FIELD(data, IH_CLK_CTRL,
704
+				     LIMIT_SMN_CLK_SOFT_OVERRIDE, field_val);
705
+		data = REG_SET_FIELD(data, IH_CLK_CTRL,
706
+				     DYN_CLK_SOFT_OVERRIDE, field_val);
707
+		data = REG_SET_FIELD(data, IH_CLK_CTRL,
708
+				     REG_CLK_SOFT_OVERRIDE, field_val);
709
+		if (def != data)
710
+			WREG32_SOC15(OSSSYS, 0, mmIH_CLK_CTRL, data);
711
+	}
712
+}
713
+
469714 static int vega10_ih_set_clockgating_state(void *handle,
470715 					  enum amd_clockgating_state state)
471716 {
717
+	struct amdgpu_device *adev = (struct amdgpu_device *)handle;
718
+
719
+	vega10_ih_update_clockgating_state(adev,
720
+				state == AMD_CG_STATE_GATE);
472721 	return 0;
722
+
473723 }
474724 
475725 static int vega10_ih_set_powergating_state(void *handle,
....@@ -497,15 +747,13 @@
497747 
498748 static const struct amdgpu_ih_funcs vega10_ih_funcs = {
499749 	.get_wptr = vega10_ih_get_wptr,
500
-	.prescreen_iv = vega10_ih_prescreen_iv,
501750 	.decode_iv = vega10_ih_decode_iv,
502751 	.set_rptr = vega10_ih_set_rptr
503752 };
504753 
505754 static void vega10_ih_set_interrupt_funcs(struct amdgpu_device *adev)
506755 {
507
-	if (adev->irq.ih_funcs == NULL)
508
-		adev->irq.ih_funcs = &vega10_ih_funcs;
756
+	adev->irq.ih_funcs = &vega10_ih_funcs;
509757 }
510758 
511759 const struct amdgpu_ip_block_version vega10_ih_ip_block =