disable FB

hc

2024-01-04 1543e317f1da31b75942316931e8f491a8920811

 kernel/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/vmmgp100.c
..
..
@@ -21,11 +21,92 @@
21
21
  */
22
22
 #include "vmm.h"
23
23
 
24
+#include <core/client.h>
24
25
 #include <subdev/fb.h>
25
26
 #include <subdev/ltc.h>
27
+#include <subdev/timer.h>
28
+#include <engine/gr.h>
26
29
 
27
30
 #include <nvif/ifc00d.h>
28
31
 #include <nvif/unpack.h>
32
+
33
+static void
34
+gp100_vmm_pfn_unmap(struct nvkm_vmm *vmm,
35
+		    struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
36
+{
37
+	struct device *dev = vmm->mmu->subdev.device->dev;
38
+	dma_addr_t addr;
39
+
40
+	nvkm_kmap(pt->memory);
41
+	while (ptes--) {
42
+		u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 0);
43
+		u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 4);
44
+		u64 data   = (u64)datahi << 32 | datalo;
45
+		if ((data & (3ULL << 1)) != 0) {
46
+			addr = (data >> 8) << 12;
47
+			dma_unmap_page(dev, addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
48
+		}
49
+		ptei++;
50
+	}
51
+	nvkm_done(pt->memory);
52
+}
53
+
54
+static bool
55
+gp100_vmm_pfn_clear(struct nvkm_vmm *vmm,
56
+		    struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
57
+{
58
+	bool dma = false;
59
+	nvkm_kmap(pt->memory);
60
+	while (ptes--) {
61
+		u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 0);
62
+		u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 8 + 4);
63
+		u64 data   = (u64)datahi << 32 | datalo;
64
+		if ((data & BIT_ULL(0)) && (data & (3ULL << 1)) != 0) {
65
+			VMM_WO064(pt, vmm, ptei * 8, data & ~BIT_ULL(0));
66
+			dma = true;
67
+		}
68
+		ptei++;
69
+	}
70
+	nvkm_done(pt->memory);
71
+	return dma;
72
+}
73
+
74
+static void
75
+gp100_vmm_pgt_pfn(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
76
+		  u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
77
+{
78
+	struct device *dev = vmm->mmu->subdev.device->dev;
79
+	dma_addr_t addr;
80
+
81
+	nvkm_kmap(pt->memory);
82
+	for (; ptes; ptes--, map->pfn++) {
83
+		u64 data = 0;
84
+
85
+		if (!(*map->pfn & NVKM_VMM_PFN_V))
86
+			continue;
87
+
88
+		if (!(*map->pfn & NVKM_VMM_PFN_W))
89
+			data |= BIT_ULL(6); /* RO. */
90
+
91
+		if (!(*map->pfn & NVKM_VMM_PFN_VRAM)) {
92
+			addr = *map->pfn >> NVKM_VMM_PFN_ADDR_SHIFT;
93
+			addr = dma_map_page(dev, pfn_to_page(addr), 0,
94
+					    PAGE_SIZE, DMA_BIDIRECTIONAL);
95
+			if (!WARN_ON(dma_mapping_error(dev, addr))) {
96
+				data |= addr >> 4;
97
+				data |= 2ULL << 1; /* SYSTEM_COHERENT_MEMORY. */
98
+				data |= BIT_ULL(3); /* VOL. */
99
+				data |= BIT_ULL(0); /* VALID. */
100
+			}
101
+		} else {
102
+			data |= (*map->pfn & NVKM_VMM_PFN_ADDR) >> 4;
103
+			data |= BIT_ULL(0); /* VALID. */
104
+		}
105
+
106
+		VMM_WO064(pt, vmm, ptei++ * 8, data);
107
+	}
108
+	nvkm_done(pt->memory);
109
+}
29
110
 
30
111
 static inline void
31
112
 gp100_vmm_pgt_pte(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
..
..
@@ -89,6 +170,9 @@
89
170
 	.mem = gp100_vmm_pgt_mem,
90
171
 	.dma = gp100_vmm_pgt_dma,
91
172
 	.sgl = gp100_vmm_pgt_sgl,
173
+	.pfn = gp100_vmm_pgt_pfn,
174
+	.pfn_clear = gp100_vmm_pfn_clear,
175
+	.pfn_unmap = gp100_vmm_pfn_unmap,
92
176
 };
93
177
 
94
178
 static void
..
..
@@ -177,12 +261,96 @@
177
261
 	VMM_FO128(pt, vmm, pdei * 0x10, 0ULL, 0ULL, pdes);
178
262
 }
179
263
 
264
+static void
265
+gp100_vmm_pd0_pfn_unmap(struct nvkm_vmm *vmm,
266
+			struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
267
+{
268
+	struct device *dev = vmm->mmu->subdev.device->dev;
269
+	dma_addr_t addr;
270
+
271
+	nvkm_kmap(pt->memory);
272
+	while (ptes--) {
273
+		u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 16 + 0);
274
+		u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 16 + 4);
275
+		u64 data   = (u64)datahi << 32 | datalo;
276
+
277
+		if ((data & (3ULL << 1)) != 0) {
278
+			addr = (data >> 8) << 12;
279
+			dma_unmap_page(dev, addr, 1UL << 21, DMA_BIDIRECTIONAL);
280
+		}
281
+		ptei++;
282
+	}
283
+	nvkm_done(pt->memory);
284
+}
285
+
286
+static bool
287
+gp100_vmm_pd0_pfn_clear(struct nvkm_vmm *vmm,
288
+			struct nvkm_mmu_pt *pt, u32 ptei, u32 ptes)
289
+{
290
+	bool dma = false;
291
+
292
+	nvkm_kmap(pt->memory);
293
+	while (ptes--) {
294
+		u32 datalo = nvkm_ro32(pt->memory, pt->base + ptei * 16 + 0);
295
+		u32 datahi = nvkm_ro32(pt->memory, pt->base + ptei * 16 + 4);
296
+		u64 data   = (u64)datahi << 32 | datalo;
297
+
298
+		if ((data & BIT_ULL(0)) && (data & (3ULL << 1)) != 0) {
299
+			VMM_WO064(pt, vmm, ptei * 16, data & ~BIT_ULL(0));
300
+			dma = true;
301
+		}
302
+		ptei++;
303
+	}
304
+	nvkm_done(pt->memory);
305
+	return dma;
306
+}
307
+
308
+static void
309
+gp100_vmm_pd0_pfn(struct nvkm_vmm *vmm, struct nvkm_mmu_pt *pt,
310
+		  u32 ptei, u32 ptes, struct nvkm_vmm_map *map)
311
+{
312
+	struct device *dev = vmm->mmu->subdev.device->dev;
313
+	dma_addr_t addr;
314
+
315
+	nvkm_kmap(pt->memory);
316
+	for (; ptes; ptes--, map->pfn++) {
317
+		u64 data = 0;
318
+
319
+		if (!(*map->pfn & NVKM_VMM_PFN_V))
320
+			continue;
321
+
322
+		if (!(*map->pfn & NVKM_VMM_PFN_W))
323
+			data |= BIT_ULL(6); /* RO. */
324
+
325
+		if (!(*map->pfn & NVKM_VMM_PFN_VRAM)) {
326
+			addr = *map->pfn >> NVKM_VMM_PFN_ADDR_SHIFT;
327
+			addr = dma_map_page(dev, pfn_to_page(addr), 0,
328
+					    1UL << 21, DMA_BIDIRECTIONAL);
329
+			if (!WARN_ON(dma_mapping_error(dev, addr))) {
330
+				data |= addr >> 4;
331
+				data |= 2ULL << 1; /* SYSTEM_COHERENT_MEMORY. */
332
+				data |= BIT_ULL(3); /* VOL. */
333
+				data |= BIT_ULL(0); /* VALID. */
334
+			}
335
+		} else {
336
+			data |= (*map->pfn & NVKM_VMM_PFN_ADDR) >> 4;
337
+			data |= BIT_ULL(0); /* VALID. */
338
+		}
339
+
340
+		VMM_WO064(pt, vmm, ptei++ * 16, data);
341
+	}
342
+	nvkm_done(pt->memory);
343
+}
344
+
180
345
 static const struct nvkm_vmm_desc_func
181
346
 gp100_vmm_desc_pd0 = {
182
347
 	.unmap = gp100_vmm_pd0_unmap,
183
348
 	.sparse = gp100_vmm_pd0_sparse,
184
349
 	.pde = gp100_vmm_pd0_pde,
185
350
 	.mem = gp100_vmm_pd0_mem,
351
+	.pfn = gp100_vmm_pd0_pfn,
352
+	.pfn_clear = gp100_vmm_pd0_pfn_clear,
353
+	.pfn_unmap = gp100_vmm_pd0_pfn_unmap,
186
354
 };
187
355
 
188
356
 static void
..
..
@@ -239,7 +407,7 @@
239
407
 	} *args = argv;
240
408
 	struct nvkm_device *device = vmm->mmu->subdev.device;
241
409
 	struct nvkm_memory *memory = map->memory;
242
-	u8  kind, priv, ro, vol;
410
+	u8  kind, kind_inv, priv, ro, vol;
243
411
 	int kindn, aper, ret = -ENOSYS;
244
412
 	const u8 *kindm;
245
413
 
..
..
@@ -266,8 +434,8 @@
266
434
 	if (WARN_ON(aper < 0))
267
435
 		return aper;
268
436
 
269
-	kindm = vmm->mmu->func->kind(vmm->mmu, &kindn);
270
-	if (kind >= kindn || kindm[kind] == 0xff) {
437
+	kindm = vmm->mmu->func->kind(vmm->mmu, &kindn, &kind_inv);
438
+	if (kind >= kindn || kindm[kind] == kind_inv) {
271
439
 		VMM_DEBUG(vmm, "kind %02x", kind);
272
440
 		return -EINVAL;
273
441
 	}
..
..
@@ -306,16 +474,99 @@
306
474
 	return 0;
307
475
 }
308
476
 
477
+static int
478
+gp100_vmm_fault_cancel(struct nvkm_vmm *vmm, void *argv, u32 argc)
479
+{
480
+	struct nvkm_device *device = vmm->mmu->subdev.device;
481
+	union {
482
+		struct gp100_vmm_fault_cancel_v0 v0;
483
+	} *args = argv;
484
+	int ret = -ENOSYS;
485
+	u32 inst, aper;
486
+
487
+	if ((ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false)))
488
+		return ret;
489
+
490
+	/* Translate MaxwellFaultBufferA instance pointer to the same
491
+	 * format as the NV_GR_FECS_CURRENT_CTX register.
492
+	 */
493
+	aper = (args->v0.inst >> 8) & 3;
494
+	args->v0.inst >>= 12;
495
+	args->v0.inst |= aper << 28;
496
+	args->v0.inst |= 0x80000000;
497
+
498
+	if (!WARN_ON(nvkm_gr_ctxsw_pause(device))) {
499
+		if ((inst = nvkm_gr_ctxsw_inst(device)) == args->v0.inst) {
500
+			gf100_vmm_invalidate(vmm, 0x0000001b
501
+					     /* CANCEL_TARGETED. */ |
502
+					     (args->v0.hub    << 20) |
503
+					     (args->v0.gpc    << 15) |
504
+					     (args->v0.client << 9));
505
+		}
506
+		WARN_ON(nvkm_gr_ctxsw_resume(device));
507
+	}
508
+
509
+	return 0;
510
+}
511
+
512
+static int
513
+gp100_vmm_fault_replay(struct nvkm_vmm *vmm, void *argv, u32 argc)
514
+{
515
+	union {
516
+		struct gp100_vmm_fault_replay_vn vn;
517
+	} *args = argv;
518
+	int ret = -ENOSYS;
519
+
520
+	if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
521
+		gf100_vmm_invalidate(vmm, 0x0000000b); /* REPLAY_GLOBAL. */
522
+	}
523
+
524
+	return ret;
525
+}
526
+
527
+int
528
+gp100_vmm_mthd(struct nvkm_vmm *vmm,
529
+	       struct nvkm_client *client, u32 mthd, void *argv, u32 argc)
530
+{
531
+	if (client->super) {
532
+		switch (mthd) {
533
+		case GP100_VMM_VN_FAULT_REPLAY:
534
+			return gp100_vmm_fault_replay(vmm, argv, argc);
535
+		case GP100_VMM_VN_FAULT_CANCEL:
536
+			return gp100_vmm_fault_cancel(vmm, argv, argc);
537
+		default:
538
+			break;
539
+		}
540
+	}
541
+	return -EINVAL;
542
+}
543
+
544
+void
545
+gp100_vmm_invalidate_pdb(struct nvkm_vmm *vmm, u64 addr)
546
+{
547
+	struct nvkm_device *device = vmm->mmu->subdev.device;
548
+	nvkm_wr32(device, 0x100cb8, lower_32_bits(addr));
549
+	nvkm_wr32(device, 0x100cec, upper_32_bits(addr));
550
+}
551
+
309
552
 void
310
553
 gp100_vmm_flush(struct nvkm_vmm *vmm, int depth)
311
554
 {
312
-	gf100_vmm_flush_(vmm, 5 /* CACHE_LEVEL_UP_TO_PDE3 */ - depth);
555
+	u32 type = (5 /* CACHE_LEVEL_UP_TO_PDE3 */ - depth) << 24;
556
+	if (atomic_read(&vmm->engref[NVKM_SUBDEV_BAR]))
557
+		type |= 0x00000004; /* HUB_ONLY */
558
+	type |= 0x00000001; /* PAGE_ALL */
559
+	gf100_vmm_invalidate(vmm, type);
313
560
 }
314
561
 
315
562
 int
316
563
 gp100_vmm_join(struct nvkm_vmm *vmm, struct nvkm_memory *inst)
317
564
 {
318
-	const u64 base = BIT_ULL(10) /* VER2 */ | BIT_ULL(11); /* 64KiB */
565
+	u64 base = BIT_ULL(10) /* VER2 */ | BIT_ULL(11) /* 64KiB */;
566
+	if (vmm->replay) {
567
+		base |= BIT_ULL(4); /* FAULT_REPLAY_TEX */
568
+		base |= BIT_ULL(5); /* FAULT_REPLAY_GCC */
569
+	}
319
570
 	return gf100_vmm_join_(vmm, inst, base);
320
571
 }
321
572
 
..
..
@@ -326,6 +577,8 @@
326
577
 	.aper = gf100_vmm_aper,
327
578
 	.valid = gp100_vmm_valid,
328
579
 	.flush = gp100_vmm_flush,
580
+	.mthd = gp100_vmm_mthd,
581
+	.invalidate_pdb = gp100_vmm_invalidate_pdb,
329
582
 	.page = {
330
583
 		{ 47, &gp100_vmm_desc_16[4], NVKM_VMM_PAGE_Sxxx },
331
584
 		{ 38, &gp100_vmm_desc_16[3], NVKM_VMM_PAGE_Sxxx },
..
..
@@ -338,10 +591,39 @@
338
591
 };
339
592
 
340
593
 int
341
-gp100_vmm_new(struct nvkm_mmu *mmu, u64 addr, u64 size, void *argv, u32 argc,
342
-	      struct lock_class_key *key, const char *name,
343
-	      struct nvkm_vmm **pvmm)
594
+gp100_vmm_new_(const struct nvkm_vmm_func *func,
595
+	       struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size,
596
+	       void *argv, u32 argc, struct lock_class_key *key,
597
+	       const char *name, struct nvkm_vmm **pvmm)
344
598
 {
345
-	return nv04_vmm_new_(&gp100_vmm, mmu, 0, addr, size,
346
-			     argv, argc, key, name, pvmm);
599
+	union {
600
+		struct gp100_vmm_vn vn;
601
+		struct gp100_vmm_v0 v0;
602
+	} *args = argv;
603
+	int ret = -ENOSYS;
604
+	bool replay;
605
+
606
+	if (!(ret = nvif_unpack(ret, &argv, &argc, args->v0, 0, 0, false))) {
607
+		replay = args->v0.fault_replay != 0;
608
+	} else
609
+	if (!(ret = nvif_unvers(ret, &argv, &argc, args->vn))) {
610
+		replay = false;
611
+	} else
612
+		return ret;
613
+
614
+	ret = nvkm_vmm_new_(func, mmu, 0, managed, addr, size, key, name, pvmm);
615
+	if (ret)
616
+		return ret;
617
+
618
+	(*pvmm)->replay = replay;
619
+	return 0;
620
+}
621
+
622
+int
623
+gp100_vmm_new(struct nvkm_mmu *mmu, bool managed, u64 addr, u64 size,
624
+	      void *argv, u32 argc, struct lock_class_key *key,
625
+	      const char *name, struct nvkm_vmm **pvmm)
626
+{
627
+	return gp100_vmm_new_(&gp100_vmm, mmu, managed, addr, size,
628
+			      argv, argc, key, name, pvmm);
347
629
 }