.. | .. |
---|
20 | 20 | * OTHER DEALINGS IN THE SOFTWARE. |
---|
21 | 21 | * |
---|
22 | 22 | */ |
---|
23 | | -#include <drm/drmP.h> |
---|
| 23 | + |
---|
| 24 | +#include <linux/pci.h> |
---|
| 25 | + |
---|
24 | 26 | #include "amdgpu.h" |
---|
25 | 27 | #include "amdgpu_ih.h" |
---|
26 | 28 | #include "vid.h" |
---|
.. | .. |
---|
99 | 101 | */ |
---|
100 | 102 | static int tonga_ih_irq_init(struct amdgpu_device *adev) |
---|
101 | 103 | { |
---|
102 | | - int rb_bufsz; |
---|
103 | 104 | u32 interrupt_cntl, ih_rb_cntl, ih_doorbell_rtpr; |
---|
104 | | - u64 wptr_off; |
---|
| 105 | + struct amdgpu_ih_ring *ih = &adev->irq.ih; |
---|
| 106 | + int rb_bufsz; |
---|
105 | 107 | |
---|
106 | 108 | /* disable irqs */ |
---|
107 | 109 | tonga_ih_disable_interrupts(adev); |
---|
.. | .. |
---|
118 | 120 | WREG32(mmINTERRUPT_CNTL, interrupt_cntl); |
---|
119 | 121 | |
---|
120 | 122 | /* Ring Buffer base. [39:8] of 40-bit address of the beginning of the ring buffer*/ |
---|
121 | | - if (adev->irq.ih.use_bus_addr) |
---|
122 | | - WREG32(mmIH_RB_BASE, adev->irq.ih.rb_dma_addr >> 8); |
---|
123 | | - else |
---|
124 | | - WREG32(mmIH_RB_BASE, adev->irq.ih.gpu_addr >> 8); |
---|
| 123 | + WREG32(mmIH_RB_BASE, ih->gpu_addr >> 8); |
---|
125 | 124 | |
---|
126 | 125 | rb_bufsz = order_base_2(adev->irq.ih.ring_size / 4); |
---|
127 | 126 | ih_rb_cntl = REG_SET_FIELD(0, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); |
---|
.. | .. |
---|
136 | 135 | WREG32(mmIH_RB_CNTL, ih_rb_cntl); |
---|
137 | 136 | |
---|
138 | 137 | /* set the writeback address whether it's enabled or not */ |
---|
139 | | - if (adev->irq.ih.use_bus_addr) |
---|
140 | | - wptr_off = adev->irq.ih.rb_dma_addr + (adev->irq.ih.wptr_offs * 4); |
---|
141 | | - else |
---|
142 | | - wptr_off = adev->wb.gpu_addr + (adev->irq.ih.wptr_offs * 4); |
---|
143 | | - WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(wptr_off)); |
---|
144 | | - WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(wptr_off) & 0xFF); |
---|
| 138 | + WREG32(mmIH_RB_WPTR_ADDR_LO, lower_32_bits(ih->wptr_addr)); |
---|
| 139 | + WREG32(mmIH_RB_WPTR_ADDR_HI, upper_32_bits(ih->wptr_addr) & 0xFF); |
---|
145 | 140 | |
---|
146 | 141 | /* set rptr, wptr to 0 */ |
---|
147 | 142 | WREG32(mmIH_RB_RPTR, 0); |
---|
.. | .. |
---|
193 | 188 | * Used by cz_irq_process(VI). |
---|
194 | 189 | * Returns the value of the wptr. |
---|
195 | 190 | */ |
---|
196 | | -static u32 tonga_ih_get_wptr(struct amdgpu_device *adev) |
---|
| 191 | +static u32 tonga_ih_get_wptr(struct amdgpu_device *adev, |
---|
| 192 | + struct amdgpu_ih_ring *ih) |
---|
197 | 193 | { |
---|
198 | 194 | u32 wptr, tmp; |
---|
199 | 195 | |
---|
200 | | - if (adev->irq.ih.use_bus_addr) |
---|
201 | | - wptr = le32_to_cpu(adev->irq.ih.ring[adev->irq.ih.wptr_offs]); |
---|
202 | | - else |
---|
203 | | - wptr = le32_to_cpu(adev->wb.wb[adev->irq.ih.wptr_offs]); |
---|
| 196 | + wptr = le32_to_cpu(*ih->wptr_cpu); |
---|
204 | 197 | |
---|
205 | | - if (REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) { |
---|
206 | | - wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); |
---|
207 | | - /* When a ring buffer overflow happen start parsing interrupt |
---|
208 | | - * from the last not overwritten vector (wptr + 16). Hopefully |
---|
209 | | - * this should allow us to catchup. |
---|
210 | | - */ |
---|
211 | | - dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", |
---|
212 | | - wptr, adev->irq.ih.rptr, (wptr + 16) & adev->irq.ih.ptr_mask); |
---|
213 | | - adev->irq.ih.rptr = (wptr + 16) & adev->irq.ih.ptr_mask; |
---|
214 | | - tmp = RREG32(mmIH_RB_CNTL); |
---|
215 | | - tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); |
---|
216 | | - WREG32(mmIH_RB_CNTL, tmp); |
---|
217 | | - } |
---|
218 | | - return (wptr & adev->irq.ih.ptr_mask); |
---|
219 | | -} |
---|
| 198 | + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) |
---|
| 199 | + goto out; |
---|
220 | 200 | |
---|
221 | | -/** |
---|
222 | | - * tonga_ih_prescreen_iv - prescreen an interrupt vector |
---|
223 | | - * |
---|
224 | | - * @adev: amdgpu_device pointer |
---|
225 | | - * |
---|
226 | | - * Returns true if the interrupt vector should be further processed. |
---|
227 | | - */ |
---|
228 | | -static bool tonga_ih_prescreen_iv(struct amdgpu_device *adev) |
---|
229 | | -{ |
---|
230 | | - u32 ring_index = adev->irq.ih.rptr >> 2; |
---|
231 | | - u16 pasid; |
---|
| 201 | + /* Double check that the overflow wasn't already cleared. */ |
---|
| 202 | + wptr = RREG32(mmIH_RB_WPTR); |
---|
232 | 203 | |
---|
233 | | - switch (le32_to_cpu(adev->irq.ih.ring[ring_index]) & 0xff) { |
---|
234 | | - case 146: |
---|
235 | | - case 147: |
---|
236 | | - pasid = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]) >> 16; |
---|
237 | | - if (!pasid || amdgpu_vm_pasid_fault_credit(adev, pasid)) |
---|
238 | | - return true; |
---|
239 | | - break; |
---|
240 | | - default: |
---|
241 | | - /* Not a VM fault */ |
---|
242 | | - return true; |
---|
243 | | - } |
---|
| 204 | + if (!REG_GET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW)) |
---|
| 205 | + goto out; |
---|
244 | 206 | |
---|
245 | | - adev->irq.ih.rptr += 16; |
---|
246 | | - return false; |
---|
| 207 | + wptr = REG_SET_FIELD(wptr, IH_RB_WPTR, RB_OVERFLOW, 0); |
---|
| 208 | + |
---|
| 209 | + /* When a ring buffer overflow happen start parsing interrupt |
---|
| 210 | + * from the last not overwritten vector (wptr + 16). Hopefully |
---|
| 211 | + * this should allow us to catchup. |
---|
| 212 | + */ |
---|
| 213 | + |
---|
| 214 | + dev_warn(adev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n", |
---|
| 215 | + wptr, ih->rptr, (wptr + 16) & ih->ptr_mask); |
---|
| 216 | + ih->rptr = (wptr + 16) & ih->ptr_mask; |
---|
| 217 | + tmp = RREG32(mmIH_RB_CNTL); |
---|
| 218 | + tmp = REG_SET_FIELD(tmp, IH_RB_CNTL, WPTR_OVERFLOW_CLEAR, 1); |
---|
| 219 | + WREG32(mmIH_RB_CNTL, tmp); |
---|
| 220 | + |
---|
| 221 | +out: |
---|
| 222 | + return (wptr & ih->ptr_mask); |
---|
247 | 223 | } |
---|
248 | 224 | |
---|
249 | 225 | /** |
---|
.. | .. |
---|
255 | 231 | * position and also advance the position. |
---|
256 | 232 | */ |
---|
257 | 233 | static void tonga_ih_decode_iv(struct amdgpu_device *adev, |
---|
258 | | - struct amdgpu_iv_entry *entry) |
---|
| 234 | + struct amdgpu_ih_ring *ih, |
---|
| 235 | + struct amdgpu_iv_entry *entry) |
---|
259 | 236 | { |
---|
260 | 237 | /* wptr/rptr are in bytes! */ |
---|
261 | | - u32 ring_index = adev->irq.ih.rptr >> 2; |
---|
| 238 | + u32 ring_index = ih->rptr >> 2; |
---|
262 | 239 | uint32_t dw[4]; |
---|
263 | 240 | |
---|
264 | | - dw[0] = le32_to_cpu(adev->irq.ih.ring[ring_index + 0]); |
---|
265 | | - dw[1] = le32_to_cpu(adev->irq.ih.ring[ring_index + 1]); |
---|
266 | | - dw[2] = le32_to_cpu(adev->irq.ih.ring[ring_index + 2]); |
---|
267 | | - dw[3] = le32_to_cpu(adev->irq.ih.ring[ring_index + 3]); |
---|
| 241 | + dw[0] = le32_to_cpu(ih->ring[ring_index + 0]); |
---|
| 242 | + dw[1] = le32_to_cpu(ih->ring[ring_index + 1]); |
---|
| 243 | + dw[2] = le32_to_cpu(ih->ring[ring_index + 2]); |
---|
| 244 | + dw[3] = le32_to_cpu(ih->ring[ring_index + 3]); |
---|
268 | 245 | |
---|
269 | | - entry->client_id = AMDGPU_IH_CLIENTID_LEGACY; |
---|
| 246 | + entry->client_id = AMDGPU_IRQ_CLIENTID_LEGACY; |
---|
270 | 247 | entry->src_id = dw[0] & 0xff; |
---|
271 | 248 | entry->src_data[0] = dw[1] & 0xfffffff; |
---|
272 | 249 | entry->ring_id = dw[2] & 0xff; |
---|
.. | .. |
---|
274 | 251 | entry->pasid = (dw[2] >> 16) & 0xffff; |
---|
275 | 252 | |
---|
276 | 253 | /* wptr/rptr are in bytes! */ |
---|
277 | | - adev->irq.ih.rptr += 16; |
---|
| 254 | + ih->rptr += 16; |
---|
278 | 255 | } |
---|
279 | 256 | |
---|
280 | 257 | /** |
---|
.. | .. |
---|
284 | 261 | * |
---|
285 | 262 | * Set the IH ring buffer rptr. |
---|
286 | 263 | */ |
---|
287 | | -static void tonga_ih_set_rptr(struct amdgpu_device *adev) |
---|
| 264 | +static void tonga_ih_set_rptr(struct amdgpu_device *adev, |
---|
| 265 | + struct amdgpu_ih_ring *ih) |
---|
288 | 266 | { |
---|
289 | | - if (adev->irq.ih.use_doorbell) { |
---|
| 267 | + if (ih->use_doorbell) { |
---|
290 | 268 | /* XXX check if swapping is necessary on BE */ |
---|
291 | | - if (adev->irq.ih.use_bus_addr) |
---|
292 | | - adev->irq.ih.ring[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr; |
---|
293 | | - else |
---|
294 | | - adev->wb.wb[adev->irq.ih.rptr_offs] = adev->irq.ih.rptr; |
---|
295 | | - WDOORBELL32(adev->irq.ih.doorbell_index, adev->irq.ih.rptr); |
---|
| 269 | + *ih->rptr_cpu = ih->rptr; |
---|
| 270 | + WDOORBELL32(ih->doorbell_index, ih->rptr); |
---|
296 | 271 | } else { |
---|
297 | | - WREG32(mmIH_RB_RPTR, adev->irq.ih.rptr); |
---|
| 272 | + WREG32(mmIH_RB_RPTR, ih->rptr); |
---|
298 | 273 | } |
---|
299 | 274 | } |
---|
300 | 275 | |
---|
.. | .. |
---|
317 | 292 | int r; |
---|
318 | 293 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
---|
319 | 294 | |
---|
320 | | - r = amdgpu_ih_ring_init(adev, 64 * 1024, true); |
---|
| 295 | + r = amdgpu_ih_ring_init(adev, &adev->irq.ih, 64 * 1024, true); |
---|
321 | 296 | if (r) |
---|
322 | 297 | return r; |
---|
323 | 298 | |
---|
324 | 299 | adev->irq.ih.use_doorbell = true; |
---|
325 | | - adev->irq.ih.doorbell_index = AMDGPU_DOORBELL_IH; |
---|
| 300 | + adev->irq.ih.doorbell_index = adev->doorbell_index.ih; |
---|
326 | 301 | |
---|
327 | 302 | r = amdgpu_irq_init(adev); |
---|
328 | 303 | |
---|
.. | .. |
---|
334 | 309 | struct amdgpu_device *adev = (struct amdgpu_device *)handle; |
---|
335 | 310 | |
---|
336 | 311 | amdgpu_irq_fini(adev); |
---|
337 | | - amdgpu_ih_ring_fini(adev); |
---|
| 312 | + amdgpu_ih_ring_fini(adev, &adev->irq.ih); |
---|
338 | 313 | amdgpu_irq_remove_domain(adev); |
---|
339 | 314 | |
---|
340 | 315 | return 0; |
---|
.. | .. |
---|
506 | 481 | |
---|
507 | 482 | static const struct amdgpu_ih_funcs tonga_ih_funcs = { |
---|
508 | 483 | .get_wptr = tonga_ih_get_wptr, |
---|
509 | | - .prescreen_iv = tonga_ih_prescreen_iv, |
---|
510 | 484 | .decode_iv = tonga_ih_decode_iv, |
---|
511 | 485 | .set_rptr = tonga_ih_set_rptr |
---|
512 | 486 | }; |
---|
513 | 487 | |
---|
514 | 488 | static void tonga_ih_set_interrupt_funcs(struct amdgpu_device *adev) |
---|
515 | 489 | { |
---|
516 | | - if (adev->irq.ih_funcs == NULL) |
---|
517 | | - adev->irq.ih_funcs = &tonga_ih_funcs; |
---|
| 490 | + adev->irq.ih_funcs = &tonga_ih_funcs; |
---|
518 | 491 | } |
---|
519 | 492 | |
---|
520 | 493 | const struct amdgpu_ip_block_version tonga_ih_ip_block = |
---|