forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-01-31 f70575805708cabdedea7498aaa3f710fde4d920 
 kernel/drivers/gpu/drm/nouveau/nouveau_drm.c
....@@ -28,9 +28,11 @@
2828 #include <linux/pci.h>
2929 #include <linux/pm_runtime.h>
3030 #include <linux/vga_switcheroo.h>
31
+#include <linux/mmu_notifier.h>
3132 
32
-#include <drm/drmP.h>
3333 #include <drm/drm_crtc_helper.h>
34
+#include <drm/drm_ioctl.h>
35
+#include <drm/drm_vblank.h>
3436 
3537 #include <core/gpuobj.h>
3638 #include <core/option.h>
....@@ -39,12 +41,12 @@
3941 
4042 #include <nvif/driver.h>
4143 #include <nvif/fifo.h>
44
+#include <nvif/push006c.h>
4245 #include <nvif/user.h>
4346 
4447 #include <nvif/class.h>
4548 #include <nvif/cl0002.h>
4649 #include <nvif/cla06f.h>
47
-#include <nvif/if0004.h>
4850 
4951 #include "nouveau_drv.h"
5052 #include "nouveau_dma.h"
....@@ -63,6 +65,8 @@
6365 #include "nouveau_usif.h"
6466 #include "nouveau_connector.h"
6567 #include "nouveau_platform.h"
68
+#include "nouveau_svm.h"
69
+#include "nouveau_dmem.h"
6670 
6771 MODULE_PARM_DESC(config, "option string to pass to driver core");
6872 static char *nouveau_config;
....@@ -120,10 +124,16 @@
120124 static inline bool
121125 nouveau_cli_work_ready(struct dma_fence *fence)
122126 {
123
-	if (!dma_fence_is_signaled(fence))
124
-		return false;
125
-	dma_fence_put(fence);
126
-	return true;
127
+	bool ret = true;
128
+
129
+	spin_lock_irq(fence->lock);
130
+	if (!dma_fence_is_signaled_locked(fence))
131
+		ret = false;
132
+	spin_unlock_irq(fence->lock);
133
+
134
+	if (ret == true)
135
+		dma_fence_put(fence);
136
+	return ret;
127137 }
128138 
129139 static void
....@@ -173,11 +183,12 @@
173183 	WARN_ON(!list_empty(&cli->worker));
174184 
175185 	usif_client_fini(cli);
186
+	nouveau_vmm_fini(&cli->svm);
176187 	nouveau_vmm_fini(&cli->vmm);
177
-	nvif_mmu_fini(&cli->mmu);
178
-	nvif_device_fini(&cli->device);
188
+	nvif_mmu_dtor(&cli->mmu);
189
+	nvif_device_dtor(&cli->device);
179190 	mutex_lock(&cli->drm->master.lock);
180
-	nvif_client_fini(&cli->base);
191
+	nvif_client_dtor(&cli->base);
181192 	mutex_unlock(&cli->drm->master.lock);
182193 }
183194 
....@@ -225,7 +236,7 @@
225236 				       cli->name, device, &cli->base);
226237 	} else {
227238 		mutex_lock(&drm->master.lock);
228
-		ret = nvif_client_init(&drm->master.base, cli->name, device,
239
+		ret = nvif_client_ctor(&drm->master.base, cli->name, device,
229240 				       &cli->base);
230241 		mutex_unlock(&drm->master.lock);
231242 	}
....@@ -234,7 +245,7 @@
234245 		goto done;
235246 	}
236247 
237
-	ret = nvif_device_init(&cli->base.object, 0, NV_DEVICE,
248
+	ret = nvif_device_ctor(&cli->base.object, "drmDevice", 0, NV_DEVICE,
238249 			       &(struct nv_device_v0) {
239250 					.device = ~0,
240251 			       }, sizeof(struct nv_device_v0),
....@@ -250,7 +261,8 @@
250261 		goto done;
251262 	}
252263 
253
-	ret = nvif_mmu_init(&cli->device.object, mmus[ret].oclass, &cli->mmu);
264
+	ret = nvif_mmu_ctor(&cli->device.object, "drmMmu", mmus[ret].oclass,
265
+			    &cli->mmu);
254266 	if (ret) {
255267 		NV_PRINTK(err, cli, "MMU allocation failed: %d\n", ret);
256268 		goto done;
....@@ -283,19 +295,132 @@
283295 }
284296 
285297 static void
286
-nouveau_accel_fini(struct nouveau_drm *drm)
298
+nouveau_accel_ce_fini(struct nouveau_drm *drm)
299
+{
300
+	nouveau_channel_idle(drm->cechan);
301
+	nvif_object_dtor(&drm->ttm.copy);
302
+	nouveau_channel_del(&drm->cechan);
303
+}
304
+
305
+static void
306
+nouveau_accel_ce_init(struct nouveau_drm *drm)
307
+{
308
+	struct nvif_device *device = &drm->client.device;
309
+	int ret = 0;
310
+
311
+	/* Allocate channel that has access to a (preferably async) copy
312
+	 * engine, to use for TTM buffer moves.
313
+	 */
314
+	if (device->info.family >= NV_DEVICE_INFO_V0_KEPLER) {
315
+		ret = nouveau_channel_new(drm, device,
316
+					  nvif_fifo_runlist_ce(device), 0,
317
+					  true, &drm->cechan);
318
+	} else
319
+	if (device->info.chipset >= 0xa3 &&
320
+	    device->info.chipset != 0xaa &&
321
+	    device->info.chipset != 0xac) {
322
+		/* Prior to Kepler, there's only a single runlist, so all
323
+		 * engines can be accessed from any channel.
324
+		 *
325
+		 * We still want to use a separate channel though.
326
+		 */
327
+		ret = nouveau_channel_new(drm, device, NvDmaFB, NvDmaTT, false,
328
+					  &drm->cechan);
329
+	}
330
+
331
+	if (ret)
332
+		NV_ERROR(drm, "failed to create ce channel, %d\n", ret);
333
+}
334
+
335
+static void
336
+nouveau_accel_gr_fini(struct nouveau_drm *drm)
287337 {
288338 	nouveau_channel_idle(drm->channel);
289
-	nvif_object_fini(&drm->ntfy);
339
+	nvif_object_dtor(&drm->ntfy);
290340 	nvkm_gpuobj_del(&drm->notify);
291
-	nvif_notify_fini(&drm->flip);
292
-	nvif_object_fini(&drm->nvsw);
293341 	nouveau_channel_del(&drm->channel);
342
+}
294343 
295
-	nouveau_channel_idle(drm->cechan);
296
-	nvif_object_fini(&drm->ttm.copy);
297
-	nouveau_channel_del(&drm->cechan);
344
+static void
345
+nouveau_accel_gr_init(struct nouveau_drm *drm)
346
+{
347
+	struct nvif_device *device = &drm->client.device;
348
+	u32 arg0, arg1;
349
+	int ret;
298350 
351
+	/* Allocate channel that has access to the graphics engine. */
352
+	if (device->info.family >= NV_DEVICE_INFO_V0_KEPLER) {
353
+		arg0 = nvif_fifo_runlist(device, NV_DEVICE_INFO_ENGINE_GR);
354
+		arg1 = 1;
355
+	} else {
356
+		arg0 = NvDmaFB;
357
+		arg1 = NvDmaTT;
358
+	}
359
+
360
+	ret = nouveau_channel_new(drm, device, arg0, arg1, false,
361
+				  &drm->channel);
362
+	if (ret) {
363
+		NV_ERROR(drm, "failed to create kernel channel, %d\n", ret);
364
+		nouveau_accel_gr_fini(drm);
365
+		return;
366
+	}
367
+
368
+	/* A SW class is used on pre-NV50 HW to assist with handling the
369
+	 * synchronisation of page flips, as well as to implement fences
370
+	 * on TNT/TNT2 HW that lacks any kind of support in host.
371
+	 */
372
+	if (!drm->channel->nvsw.client && device->info.family < NV_DEVICE_INFO_V0_TESLA) {
373
+		ret = nvif_object_ctor(&drm->channel->user, "drmNvsw",
374
+				       NVDRM_NVSW, nouveau_abi16_swclass(drm),
375
+				       NULL, 0, &drm->channel->nvsw);
376
+		if (ret == 0) {
377
+			struct nvif_push *push = drm->channel->chan.push;
378
+			ret = PUSH_WAIT(push, 2);
379
+			if (ret == 0)
380
+				PUSH_NVSQ(push, NV_SW, 0x0000, drm->channel->nvsw.handle);
381
+		}
382
+
383
+		if (ret) {
384
+			NV_ERROR(drm, "failed to allocate sw class, %d\n", ret);
385
+			nouveau_accel_gr_fini(drm);
386
+			return;
387
+		}
388
+	}
389
+
390
+	/* NvMemoryToMemoryFormat requires a notifier ctxdma for some reason,
391
+	 * even if notification is never requested, so, allocate a ctxdma on
392
+	 * any GPU where it's possible we'll end up using M2MF for BO moves.
393
+	 */
394
+	if (device->info.family < NV_DEVICE_INFO_V0_FERMI) {
395
+		ret = nvkm_gpuobj_new(nvxx_device(device), 32, 0, false, NULL,
396
+				      &drm->notify);
397
+		if (ret) {
398
+			NV_ERROR(drm, "failed to allocate notifier, %d\n", ret);
399
+			nouveau_accel_gr_fini(drm);
400
+			return;
401
+		}
402
+
403
+		ret = nvif_object_ctor(&drm->channel->user, "drmM2mfNtfy",
404
+				       NvNotify0, NV_DMA_IN_MEMORY,
405
+				       &(struct nv_dma_v0) {
406
+						.target = NV_DMA_V0_TARGET_VRAM,
407
+						.access = NV_DMA_V0_ACCESS_RDWR,
408
+						.start = drm->notify->addr,
409
+						.limit = drm->notify->addr + 31
410
+				       }, sizeof(struct nv_dma_v0),
411
+				       &drm->ntfy);
412
+		if (ret) {
413
+			nouveau_accel_gr_fini(drm);
414
+			return;
415
+		}
416
+	}
417
+}
418
+
419
+static void
420
+nouveau_accel_fini(struct nouveau_drm *drm)
421
+{
422
+	nouveau_accel_ce_fini(drm);
423
+	nouveau_accel_gr_fini(drm);
299424 	if (drm->fence)
300425 		nouveau_fence(drm)->dtor(drm);
301426 }
....@@ -305,23 +430,16 @@
305430 {
306431 	struct nvif_device *device = &drm->client.device;
307432 	struct nvif_sclass *sclass;
308
-	u32 arg0, arg1;
309433 	int ret, i, n;
310434 
311435 	if (nouveau_noaccel)
312436 		return;
313437 
438
+	/* Initialise global support for channels, and synchronisation. */
314439 	ret = nouveau_channels_init(drm);
315440 	if (ret)
316441 		return;
317442 
318
-	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_VOLTA) {
319
-		ret = nvif_user_init(device);
320
-		if (ret)
321
-			return;
322
-	}
323
-
324
-	/* initialise synchronisation routines */
325443 	/*XXX: this is crap, but the fence/channel stuff is a little
326444 	 *     backwards in some places.  this will be fixed.
327445 	 */
....@@ -353,6 +471,7 @@
353471 		case MAXWELL_CHANNEL_GPFIFO_A:
354472 		case PASCAL_CHANNEL_GPFIFO_A:
355473 		case VOLTA_CHANNEL_GPFIFO_A:
474
+		case TURING_CHANNEL_GPFIFO_A:
356475 			ret = nvc0_fence_create(drm);
357476 			break;
358477 		default:
....@@ -367,166 +486,57 @@
367486 		return;
368487 	}
369488 
370
-	if (device->info.family >= NV_DEVICE_INFO_V0_KEPLER) {
371
-		ret = nouveau_channel_new(drm, &drm->client.device,
372
-					  nvif_fifo_runlist_ce(device), 0,
373
-					  &drm->cechan);
489
+	/* Volta requires access to a doorbell register for kickoff. */
490
+	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_VOLTA) {
491
+		ret = nvif_user_ctor(device, "drmUsermode");
374492 		if (ret)
375
-			NV_ERROR(drm, "failed to create ce channel, %d\n", ret);
376
-
377
-		arg0 = nvif_fifo_runlist(device, NV_DEVICE_INFO_ENGINE_GR);
378
-		arg1 = 1;
379
-	} else
380
-	if (device->info.chipset >= 0xa3 &&
381
-	    device->info.chipset != 0xaa &&
382
-	    device->info.chipset != 0xac) {
383
-		ret = nouveau_channel_new(drm, &drm->client.device,
384
-					  NvDmaFB, NvDmaTT, &drm->cechan);
385
-		if (ret)
386
-			NV_ERROR(drm, "failed to create ce channel, %d\n", ret);
387
-
388
-		arg0 = NvDmaFB;
389
-		arg1 = NvDmaTT;
390
-	} else {
391
-		arg0 = NvDmaFB;
392
-		arg1 = NvDmaTT;
393
-	}
394
-
395
-	ret = nouveau_channel_new(drm, &drm->client.device,
396
-				  arg0, arg1, &drm->channel);
397
-	if (ret) {
398
-		NV_ERROR(drm, "failed to create kernel channel, %d\n", ret);
399
-		nouveau_accel_fini(drm);
400
-		return;
401
-	}
402
-
403
-	if (device->info.family < NV_DEVICE_INFO_V0_TESLA) {
404
-		ret = nvif_object_init(&drm->channel->user, NVDRM_NVSW,
405
-				       nouveau_abi16_swclass(drm), NULL, 0,
406
-				       &drm->nvsw);
407
-		if (ret == 0) {
408
-			ret = RING_SPACE(drm->channel, 2);
409
-			if (ret == 0) {
410
-				BEGIN_NV04(drm->channel, NvSubSw, 0, 1);
411
-				OUT_RING  (drm->channel, drm->nvsw.handle);
412
-			}
413
-
414
-			ret = nvif_notify_init(&drm->nvsw,
415
-					       nouveau_flip_complete,
416
-					       false, NV04_NVSW_NTFY_UEVENT,
417
-					       NULL, 0, 0, &drm->flip);
418
-			if (ret == 0)
419
-				ret = nvif_notify_get(&drm->flip);
420
-			if (ret) {
421
-				nouveau_accel_fini(drm);
422
-				return;
423
-			}
424
-		}
425
-
426
-		if (ret) {
427
-			NV_ERROR(drm, "failed to allocate sw class, %d\n", ret);
428
-			nouveau_accel_fini(drm);
429493 			return;
430
-		}
431494 	}
432495 
433
-	if (device->info.family < NV_DEVICE_INFO_V0_FERMI) {
434
-		ret = nvkm_gpuobj_new(nvxx_device(&drm->client.device), 32, 0,
435
-				      false, NULL, &drm->notify);
436
-		if (ret) {
437
-			NV_ERROR(drm, "failed to allocate notifier, %d\n", ret);
438
-			nouveau_accel_fini(drm);
439
-			return;
440
-		}
496
+	/* Allocate channels we need to support various functions. */
497
+	nouveau_accel_gr_init(drm);
498
+	nouveau_accel_ce_init(drm);
441499 
442
-		ret = nvif_object_init(&drm->channel->user, NvNotify0,
443
-				       NV_DMA_IN_MEMORY,
444
-				       &(struct nv_dma_v0) {
445
-						.target = NV_DMA_V0_TARGET_VRAM,
446
-						.access = NV_DMA_V0_ACCESS_RDWR,
447
-						.start = drm->notify->addr,
448
-						.limit = drm->notify->addr + 31
449
-				       }, sizeof(struct nv_dma_v0),
450
-				       &drm->ntfy);
451
-		if (ret) {
452
-			nouveau_accel_fini(drm);
453
-			return;
454
-		}
455
-	}
456
-
457
-
500
+	/* Initialise accelerated TTM buffer moves. */
458501 	nouveau_bo_move_init(drm);
459502 }
460503 
461
-static int nouveau_drm_probe(struct pci_dev *pdev,
462
-			     const struct pci_device_id *pent)
504
+static void __printf(2, 3)
505
+nouveau_drm_errorf(struct nvif_object *object, const char *fmt, ...)
463506 {
464
-	struct nvkm_device *device;
465
-	struct apertures_struct *aper;
466
-	bool boot = false;
467
-	int ret;
507
+	struct nouveau_drm *drm = container_of(object->parent, typeof(*drm), parent);
508
+	struct va_format vaf;
509
+	va_list va;
468510 
469
-	if (vga_switcheroo_client_probe_defer(pdev))
470
-		return -EPROBE_DEFER;
471
-
472
-	/* We need to check that the chipset is supported before booting
473
-	 * fbdev off the hardware, as there's no way to put it back.
474
-	 */
475
-	ret = nvkm_device_pci_new(pdev, NULL, "error", true, false, 0, &device);
476
-	if (ret)
477
-		return ret;
478
-
479
-	nvkm_device_del(&device);
480
-
481
-	/* Remove conflicting drivers (vesafb, efifb etc). */
482
-	aper = alloc_apertures(3);
483
-	if (!aper)
484
-		return -ENOMEM;
485
-
486
-	aper->ranges[0].base = pci_resource_start(pdev, 1);
487
-	aper->ranges[0].size = pci_resource_len(pdev, 1);
488
-	aper->count = 1;
489
-
490
-	if (pci_resource_len(pdev, 2)) {
491
-		aper->ranges[aper->count].base = pci_resource_start(pdev, 2);
492
-		aper->ranges[aper->count].size = pci_resource_len(pdev, 2);
493
-		aper->count++;
494
-	}
495
-
496
-	if (pci_resource_len(pdev, 3)) {
497
-		aper->ranges[aper->count].base = pci_resource_start(pdev, 3);
498
-		aper->ranges[aper->count].size = pci_resource_len(pdev, 3);
499
-		aper->count++;
500
-	}
501
-
502
-#ifdef CONFIG_X86
503
-	boot = pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
504
-#endif
505
-	if (nouveau_modeset != 2)
506
-		drm_fb_helper_remove_conflicting_framebuffers(aper, "nouveaufb", boot);
507
-	kfree(aper);
508
-
509
-	ret = nvkm_device_pci_new(pdev, nouveau_config, nouveau_debug,
510
-				  true, true, ~0ULL, &device);
511
-	if (ret)
512
-		return ret;
513
-
514
-	pci_set_master(pdev);
515
-
516
-	if (nouveau_atomic)
517
-		driver_pci.driver_features |= DRIVER_ATOMIC;
518
-
519
-	ret = drm_get_pci_dev(pdev, pent, &driver_pci);
520
-	if (ret) {
521
-		nvkm_device_del(&device);
522
-		return ret;
523
-	}
524
-
525
-	return 0;
511
+	va_start(va, fmt);
512
+	vaf.fmt = fmt;
513
+	vaf.va = &va;
514
+	NV_ERROR(drm, "%pV", &vaf);
515
+	va_end(va);
526516 }
527517 
518
+static void __printf(2, 3)
519
+nouveau_drm_debugf(struct nvif_object *object, const char *fmt, ...)
520
+{
521
+	struct nouveau_drm *drm = container_of(object->parent, typeof(*drm), parent);
522
+	struct va_format vaf;
523
+	va_list va;
524
+
525
+	va_start(va, fmt);
526
+	vaf.fmt = fmt;
527
+	vaf.va = &va;
528
+	NV_DEBUG(drm, "%pV", &vaf);
529
+	va_end(va);
530
+}
531
+
532
+static const struct nvif_parent_func
533
+nouveau_parent = {
534
+	.debugf = nouveau_drm_debugf,
535
+	.errorf = nouveau_drm_errorf,
536
+};
537
+
528538 static int
529
-nouveau_drm_load(struct drm_device *dev, unsigned long flags)
539
+nouveau_drm_device_init(struct drm_device *dev)
530540 {
531541 	struct nouveau_drm *drm;
532542 	int ret;
....@@ -536,13 +546,16 @@
536546 	dev->dev_private = drm;
537547 	drm->dev = dev;
538548 
549
+	nvif_parent_ctor(&nouveau_parent, &drm->parent);
550
+	drm->master.base.object.parent = &drm->parent;
551
+
539552 	ret = nouveau_cli_init(drm, "DRM-master", &drm->master);
540553 	if (ret)
541
-		return ret;
554
+		goto fail_alloc;
542555 
543556 	ret = nouveau_cli_init(drm, "DRM", &drm->client);
544557 	if (ret)
545
-		return ret;
558
+		goto fail_master;
546559 
547560 	dev->irq_enabled = true;
548561 
....@@ -550,6 +563,7 @@
550563 		nvkm_dbgopt(nouveau_debug, "DRM");
551564 
552565 	INIT_LIST_HEAD(&drm->clients);
566
+	mutex_init(&drm->clients_lock);
553567 	spin_lock_init(&drm->tile.lock);
554568 
555569 	/* workaround an odd issue on nvc1 by disabling the device's
....@@ -569,19 +583,22 @@
569583 	if (ret)
570584 		goto fail_bios;
571585 
586
+	nouveau_accel_init(drm);
587
+
572588 	ret = nouveau_display_create(dev);
573589 	if (ret)
574590 		goto fail_dispctor;
575591 
576592 	if (dev->mode_config.num_crtc) {
577
-		ret = nouveau_display_init(dev);
593
+		ret = nouveau_display_init(dev, false, false);
578594 		if (ret)
579595 			goto fail_dispinit;
580596 	}
581597 
582598 	nouveau_debugfs_init(drm);
583599 	nouveau_hwmon_init(dev);
584
-	nouveau_accel_init(drm);
600
+	nouveau_svm_init(drm);
601
+	nouveau_dmem_init(drm);
585602 	nouveau_fbcon_init(dev);
586603 	nouveau_led_init(dev);
587604 
....@@ -599,20 +616,25 @@
599616 fail_dispinit:
600617 	nouveau_display_destroy(dev);
601618 fail_dispctor:
619
+	nouveau_accel_fini(drm);
602620 	nouveau_bios_takedown(dev);
603621 fail_bios:
604622 	nouveau_ttm_fini(drm);
605623 fail_ttm:
606624 	nouveau_vga_fini(drm);
607625 	nouveau_cli_fini(&drm->client);
626
+fail_master:
608627 	nouveau_cli_fini(&drm->master);
628
+fail_alloc:
629
+	nvif_parent_dtor(&drm->parent);
609630 	kfree(drm);
610631 	return ret;
611632 }
612633 
613634 static void
614
-nouveau_drm_unload(struct drm_device *dev)
635
+nouveau_drm_device_fini(struct drm_device *dev)
615636 {
637
+	struct nouveau_cli *cli, *temp_cli;
616638 	struct nouveau_drm *drm = nouveau_drm(dev);
617639 
618640 	if (nouveau_pmops_runtime()) {
....@@ -622,7 +644,8 @@
622644 
623645 	nouveau_led_fini(dev);
624646 	nouveau_fbcon_fini(dev);
625
-	nouveau_accel_fini(drm);
647
+	nouveau_dmem_fini(drm);
648
+	nouveau_svm_fini(drm);
626649 	nouveau_hwmon_fini(dev);
627650 	nouveau_debugfs_fini(drm);
628651 
....@@ -630,14 +653,163 @@
630653 		nouveau_display_fini(dev, false, false);
631654 	nouveau_display_destroy(dev);
632655 
656
+	nouveau_accel_fini(drm);
633657 	nouveau_bios_takedown(dev);
634658 
635659 	nouveau_ttm_fini(drm);
636660 	nouveau_vga_fini(drm);
637661 
662
+	/*
663
+	 * There may be existing clients from as-yet unclosed files. For now,
664
+	 * clean them up here rather than deferring until the file is closed,
665
+	 * but this likely not correct if we want to support hot-unplugging
666
+	 * properly.
667
+	 */
668
+	mutex_lock(&drm->clients_lock);
669
+	list_for_each_entry_safe(cli, temp_cli, &drm->clients, head) {
670
+		list_del(&cli->head);
671
+		mutex_lock(&cli->mutex);
672
+		if (cli->abi16)
673
+			nouveau_abi16_fini(cli->abi16);
674
+		mutex_unlock(&cli->mutex);
675
+		nouveau_cli_fini(cli);
676
+		kfree(cli);
677
+	}
678
+	mutex_unlock(&drm->clients_lock);
679
+
638680 	nouveau_cli_fini(&drm->client);
639681 	nouveau_cli_fini(&drm->master);
682
+	nvif_parent_dtor(&drm->parent);
683
+	mutex_destroy(&drm->clients_lock);
640684 	kfree(drm);
685
+}
686
+
687
+/*
688
+ * On some Intel PCIe bridge controllers doing a
689
+ * D0 -> D3hot -> D3cold -> D0 sequence causes Nvidia GPUs to not reappear.
690
+ * Skipping the intermediate D3hot step seems to make it work again. This is
691
+ * probably caused by not meeting the expectation the involved AML code has
692
+ * when the GPU is put into D3hot state before invoking it.
693
+ *
694
+ * This leads to various manifestations of this issue:
695
+ *  - AML code execution to power on the GPU hits an infinite loop (as the
696
+ *    code waits on device memory to change).
697
+ *  - kernel crashes, as all PCI reads return -1, which most code isn't able
698
+ *    to handle well enough.
699
+ *
700
+ * In all cases dmesg will contain at least one line like this:
701
+ * 'nouveau 0000:01:00.0: Refused to change power state, currently in D3'
702
+ * followed by a lot of nouveau timeouts.
703
+ *
704
+ * In the \_SB.PCI0.PEG0.PG00._OFF code deeper down writes bit 0x80 to the not
705
+ * documented PCI config space register 0x248 of the Intel PCIe bridge
706
+ * controller (0x1901) in order to change the state of the PCIe link between
707
+ * the PCIe port and the GPU. There are alternative code paths using other
708
+ * registers, which seem to work fine (executed pre Windows 8):
709
+ *  - 0xbc bit 0x20 (publicly available documentation claims 'reserved')
710
+ *  - 0xb0 bit 0x10 (link disable)
711
+ * Changing the conditions inside the firmware by poking into the relevant
712
+ * addresses does resolve the issue, but it seemed to be ACPI private memory
713
+ * and not any device accessible memory at all, so there is no portable way of
714
+ * changing the conditions.
715
+ * On a XPS 9560 that means bits [0,3] on \CPEX need to be cleared.
716
+ *
717
+ * The only systems where this behavior can be seen are hybrid graphics laptops
718
+ * with a secondary Nvidia Maxwell, Pascal or Turing GPU. It's unclear whether
719
+ * this issue only occurs in combination with listed Intel PCIe bridge
720
+ * controllers and the mentioned GPUs or other devices as well.
721
+ *
722
+ * documentation on the PCIe bridge controller can be found in the
723
+ * "7th Generation Intel® Processor Families for H Platforms Datasheet Volume 2"
724
+ * Section "12 PCI Express* Controller (x16) Registers"
725
+ */
726
+
727
+static void quirk_broken_nv_runpm(struct pci_dev *pdev)
728
+{
729
+	struct drm_device *dev = pci_get_drvdata(pdev);
730
+	struct nouveau_drm *drm = nouveau_drm(dev);
731
+	struct pci_dev *bridge = pci_upstream_bridge(pdev);
732
+
733
+	if (!bridge || bridge->vendor != PCI_VENDOR_ID_INTEL)
734
+		return;
735
+
736
+	switch (bridge->device) {
737
+	case 0x1901:
738
+		drm->old_pm_cap = pdev->pm_cap;
739
+		pdev->pm_cap = 0;
740
+		NV_INFO(drm, "Disabling PCI power management to avoid bug\n");
741
+		break;
742
+	}
743
+}
744
+
745
+static int nouveau_drm_probe(struct pci_dev *pdev,
746
+			     const struct pci_device_id *pent)
747
+{
748
+	struct nvkm_device *device;
749
+	struct drm_device *drm_dev;
750
+	int ret;
751
+
752
+	if (vga_switcheroo_client_probe_defer(pdev))
753
+		return -EPROBE_DEFER;
754
+
755
+	/* We need to check that the chipset is supported before booting
756
+	 * fbdev off the hardware, as there's no way to put it back.
757
+	 */
758
+	ret = nvkm_device_pci_new(pdev, nouveau_config, "error",
759
+				  true, false, 0, &device);
760
+	if (ret)
761
+		return ret;
762
+
763
+	nvkm_device_del(&device);
764
+
765
+	/* Remove conflicting drivers (vesafb, efifb etc). */
766
+	ret = drm_fb_helper_remove_conflicting_pci_framebuffers(pdev, "nouveaufb");
767
+	if (ret)
768
+		return ret;
769
+
770
+	ret = nvkm_device_pci_new(pdev, nouveau_config, nouveau_debug,
771
+				  true, true, ~0ULL, &device);
772
+	if (ret)
773
+		return ret;
774
+
775
+	pci_set_master(pdev);
776
+
777
+	if (nouveau_atomic)
778
+		driver_pci.driver_features |= DRIVER_ATOMIC;
779
+
780
+	drm_dev = drm_dev_alloc(&driver_pci, &pdev->dev);
781
+	if (IS_ERR(drm_dev)) {
782
+		ret = PTR_ERR(drm_dev);
783
+		goto fail_nvkm;
784
+	}
785
+
786
+	ret = pci_enable_device(pdev);
787
+	if (ret)
788
+		goto fail_drm;
789
+
790
+	drm_dev->pdev = pdev;
791
+	pci_set_drvdata(pdev, drm_dev);
792
+
793
+	ret = nouveau_drm_device_init(drm_dev);
794
+	if (ret)
795
+		goto fail_pci;
796
+
797
+	ret = drm_dev_register(drm_dev, pent->driver_data);
798
+	if (ret)
799
+		goto fail_drm_dev_init;
800
+
801
+	quirk_broken_nv_runpm(pdev);
802
+	return 0;
803
+
804
+fail_drm_dev_init:
805
+	nouveau_drm_device_fini(drm_dev);
806
+fail_pci:
807
+	pci_disable_device(pdev);
808
+fail_drm:
809
+	drm_dev_put(drm_dev);
810
+fail_nvkm:
811
+	nvkm_device_del(&device);
812
+	return ret;
641813 }
642814 
643815 void
....@@ -647,11 +819,14 @@
647819 	struct nvkm_client *client;
648820 	struct nvkm_device *device;
649821 
822
+	drm_dev_unplug(dev);
823
+
650824 	dev->irq_enabled = false;
651825 	client = nvxx_client(&drm->client.base);
652826 	device = nvkm_device_find(client->device);
653
-	drm_put_dev(dev);
654827 
828
+	nouveau_drm_device_fini(dev);
829
+	drm_dev_put(dev);
655830 	nvkm_device_del(&device);
656831 }
657832 
....@@ -659,8 +834,13 @@
659834 nouveau_drm_remove(struct pci_dev *pdev)
660835 {
661836 	struct drm_device *dev = pci_get_drvdata(pdev);
837
+	struct nouveau_drm *drm = nouveau_drm(dev);
662838 
839
+	/* revert our workaround */
840
+	if (drm->old_pm_cap)
841
+		pdev->pm_cap = drm->old_pm_cap;
663842 	nouveau_drm_device_remove(dev);
843
+	pci_disable_device(pdev);
664844 }
665845 
666846 static int
....@@ -669,6 +849,8 @@
669849 	struct nouveau_drm *drm = nouveau_drm(dev);
670850 	int ret;
671851 
852
+	nouveau_svm_suspend(drm);
853
+	nouveau_dmem_suspend(drm);
672854 	nouveau_led_suspend(dev);
673855 
674856 	if (dev->mode_config.num_crtc) {
....@@ -726,10 +908,15 @@
726908 static int
727909 nouveau_do_resume(struct drm_device *dev, bool runtime)
728910 {
911
+	int ret = 0;
729912 	struct nouveau_drm *drm = nouveau_drm(dev);
730913 
731914 	NV_DEBUG(drm, "resuming object tree...\n");
732
-	nvif_client_resume(&drm->master.base);
915
+	ret = nvif_client_resume(&drm->master.base);
916
+	if (ret) {
917
+		NV_ERROR(drm, "Client resume failed with error: %d\n", ret);
918
+		return ret;
919
+	}
733920 
734921 	NV_DEBUG(drm, "resuming fence...\n");
735922 	if (drm->fence && nouveau_fence(drm)->resume)
....@@ -745,7 +932,8 @@
745932 	}
746933 
747934 	nouveau_led_resume(dev);
748
-
935
+	nouveau_dmem_resume(drm);
936
+	nouveau_svm_resume(drm);
749937 	return 0;
750938 }
751939 
....@@ -792,7 +980,7 @@
792980 	ret = nouveau_do_resume(drm_dev, false);
793981 
794982 	/* Monitors may have been connected / disconnected during suspend */
795
-	schedule_work(&nouveau_drm(drm_dev)->hpd_work);
983
+	nouveau_display_hpd_resume(drm_dev);
796984 
797985 	return ret;
798986 }
....@@ -848,6 +1036,7 @@
8481036 {
8491037 	struct pci_dev *pdev = to_pci_dev(dev);
8501038 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
1039
+	struct nouveau_drm *drm = nouveau_drm(drm_dev);
8511040 	struct nvif_device *device = &nouveau_drm(drm_dev)->client.device;
8521041 	int ret;
8531042 
....@@ -864,13 +1053,17 @@
8641053 	pci_set_master(pdev);
8651054 
8661055 	ret = nouveau_do_resume(drm_dev, true);
1056
+	if (ret) {
1057
+		NV_ERROR(drm, "resume failed with: %d\n", ret);
1058
+		return ret;
1059
+	}
8671060 
8681061 	/* do magic */
8691062 	nvif_mask(&device->object, 0x088488, (1 << 25), (1 << 25));
8701063 	drm_dev->switch_power_state = DRM_SWITCH_POWER_ON;
8711064 
8721065 	/* Monitors may have been connected / disconnected during suspend */
873
-	schedule_work(&nouveau_drm(drm_dev)->hpd_work);
1066
+	nouveau_display_hpd_resume(drm_dev);
8741067 
8751068 	return ret;
8761069 }
....@@ -920,9 +1113,9 @@
9201113 
9211114 	fpriv->driver_priv = cli;
9221115 
923
-	mutex_lock(&drm->client.mutex);
1116
+	mutex_lock(&drm->clients_lock);
9241117 	list_add(&cli->head, &drm->clients);
925
-	mutex_unlock(&drm->client.mutex);
1118
+	mutex_unlock(&drm->clients_lock);
9261119 
9271120 done:
9281121 	if (ret && cli) {
....@@ -940,6 +1133,16 @@
9401133 {
9411134 	struct nouveau_cli *cli = nouveau_cli(fpriv);
9421135 	struct nouveau_drm *drm = nouveau_drm(dev);
1136
+	int dev_index;
1137
+
1138
+	/*
1139
+	 * The device is gone, and as it currently stands all clients are
1140
+	 * cleaned up in the removal codepath. In the future this may change
1141
+	 * so that we can support hot-unplugging, but for now we immediately
1142
+	 * return to avoid a double-free situation.
1143
+	 */
1144
+	if (!drm_dev_enter(dev, &dev_index))
1145
+		return;
9431146 
9441147 	pm_runtime_get_sync(dev->dev);
9451148 
....@@ -948,30 +1151,33 @@
9481151 		nouveau_abi16_fini(cli->abi16);
9491152 	mutex_unlock(&cli->mutex);
9501153 
951
-	mutex_lock(&drm->client.mutex);
1154
+	mutex_lock(&drm->clients_lock);
9521155 	list_del(&cli->head);
953
-	mutex_unlock(&drm->client.mutex);
1156
+	mutex_unlock(&drm->clients_lock);
9541157 
9551158 	nouveau_cli_fini(cli);
9561159 	kfree(cli);
9571160 	pm_runtime_mark_last_busy(dev->dev);
9581161 	pm_runtime_put_autosuspend(dev->dev);
1162
+	drm_dev_exit(dev_index);
9591163 }
9601164 
9611165 static const struct drm_ioctl_desc
9621166 nouveau_ioctls[] = {
963
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GETPARAM, nouveau_abi16_ioctl_getparam, DRM_AUTH|DRM_RENDER_ALLOW),
964
-	DRM_IOCTL_DEF_DRV(NOUVEAU_SETPARAM, nouveau_abi16_ioctl_setparam, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
965
-	DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_ALLOC, nouveau_abi16_ioctl_channel_alloc, DRM_AUTH|DRM_RENDER_ALLOW),
966
-	DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_FREE, nouveau_abi16_ioctl_channel_free, DRM_AUTH|DRM_RENDER_ALLOW),
967
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GROBJ_ALLOC, nouveau_abi16_ioctl_grobj_alloc, DRM_AUTH|DRM_RENDER_ALLOW),
968
-	DRM_IOCTL_DEF_DRV(NOUVEAU_NOTIFIEROBJ_ALLOC, nouveau_abi16_ioctl_notifierobj_alloc, DRM_AUTH|DRM_RENDER_ALLOW),
969
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GPUOBJ_FREE, nouveau_abi16_ioctl_gpuobj_free, DRM_AUTH|DRM_RENDER_ALLOW),
970
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_NEW, nouveau_gem_ioctl_new, DRM_AUTH|DRM_RENDER_ALLOW),
971
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_PUSHBUF, nouveau_gem_ioctl_pushbuf, DRM_AUTH|DRM_RENDER_ALLOW),
972
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_PREP, nouveau_gem_ioctl_cpu_prep, DRM_AUTH|DRM_RENDER_ALLOW),
973
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_FINI, nouveau_gem_ioctl_cpu_fini, DRM_AUTH|DRM_RENDER_ALLOW),
974
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_INFO, nouveau_gem_ioctl_info, DRM_AUTH|DRM_RENDER_ALLOW),
1167
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GETPARAM, nouveau_abi16_ioctl_getparam, DRM_RENDER_ALLOW),
1168
+	DRM_IOCTL_DEF_DRV(NOUVEAU_SETPARAM, drm_invalid_op, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1169
+	DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_ALLOC, nouveau_abi16_ioctl_channel_alloc, DRM_RENDER_ALLOW),
1170
+	DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_FREE, nouveau_abi16_ioctl_channel_free, DRM_RENDER_ALLOW),
1171
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GROBJ_ALLOC, nouveau_abi16_ioctl_grobj_alloc, DRM_RENDER_ALLOW),
1172
+	DRM_IOCTL_DEF_DRV(NOUVEAU_NOTIFIEROBJ_ALLOC, nouveau_abi16_ioctl_notifierobj_alloc, DRM_RENDER_ALLOW),
1173
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GPUOBJ_FREE, nouveau_abi16_ioctl_gpuobj_free, DRM_RENDER_ALLOW),
1174
+	DRM_IOCTL_DEF_DRV(NOUVEAU_SVM_INIT, nouveau_svmm_init, DRM_RENDER_ALLOW),
1175
+	DRM_IOCTL_DEF_DRV(NOUVEAU_SVM_BIND, nouveau_svmm_bind, DRM_RENDER_ALLOW),
1176
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_NEW, nouveau_gem_ioctl_new, DRM_RENDER_ALLOW),
1177
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_PUSHBUF, nouveau_gem_ioctl_pushbuf, DRM_RENDER_ALLOW),
1178
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_PREP, nouveau_gem_ioctl_cpu_prep, DRM_RENDER_ALLOW),
1179
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_FINI, nouveau_gem_ioctl_cpu_fini, DRM_RENDER_ALLOW),
1180
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_INFO, nouveau_gem_ioctl_info, DRM_RENDER_ALLOW),
9751181 };
9761182 
9771183 long
....@@ -1019,14 +1225,12 @@
10191225 static struct drm_driver
10201226 driver_stub = {
10211227 	.driver_features =
1022
-		DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME | DRIVER_RENDER
1228
+		DRIVER_GEM | DRIVER_MODESET | DRIVER_RENDER
10231229 #if defined(CONFIG_NOUVEAU_LEGACY_CTX_SUPPORT)
10241230 		| DRIVER_KMS_LEGACY_CONTEXT
10251231 #endif
10261232 		,
10271233 
1028
-	.load = nouveau_drm_load,
1029
-	.unload = nouveau_drm_unload,
10301234 	.open = nouveau_drm_open,
10311235 	.postclose = nouveau_drm_postclose,
10321236 	.lastclose = nouveau_vga_lastclose,
....@@ -1035,21 +1239,13 @@
10351239 	.debugfs_init = nouveau_drm_debugfs_init,
10361240 #endif
10371241 
1038
-	.enable_vblank = nouveau_display_vblank_enable,
1039
-	.disable_vblank = nouveau_display_vblank_disable,
1040
-	.get_scanout_position = nouveau_display_scanoutpos,
1041
-	.get_vblank_timestamp = drm_calc_vbltimestamp_from_scanoutpos,
1042
-
10431242 	.ioctls = nouveau_ioctls,
10441243 	.num_ioctls = ARRAY_SIZE(nouveau_ioctls),
10451244 	.fops = &nouveau_driver_fops,
10461245 
10471246 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
10481247 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
1049
-	.gem_prime_export = drm_gem_prime_export,
1050
-	.gem_prime_import = drm_gem_prime_import,
10511248 	.gem_prime_pin = nouveau_gem_prime_pin,
1052
-	.gem_prime_res_obj = nouveau_gem_prime_res_obj,
10531249 	.gem_prime_unpin = nouveau_gem_prime_unpin,
10541250 	.gem_prime_get_sg_table = nouveau_gem_prime_get_sg_table,
10551251 	.gem_prime_import_sg_table = nouveau_gem_prime_import_sg_table,
....@@ -1147,10 +1343,16 @@
11471343 		goto err_free;
11481344 	}
11491345 
1346
+	err = nouveau_drm_device_init(drm);
1347
+	if (err)
1348
+		goto err_put;
1349
+
11501350 	platform_set_drvdata(pdev, drm);
11511351 
11521352 	return drm;
11531353 
1354
+err_put:
1355
+	drm_dev_put(drm);
11541356 err_free:
11551357 	nvkm_device_del(pdevice);
11561358 
....@@ -1202,6 +1404,8 @@
12021404 #ifdef CONFIG_NOUVEAU_PLATFORM_DRIVER
12031405 	platform_driver_unregister(&nouveau_platform_driver);
12041406 #endif
1407
+	if (IS_ENABLED(CONFIG_DRM_NOUVEAU_SVM))
1408
+		mmu_notifier_synchronize();
12051409 }
12061410 
12071411 module_init(nouveau_drm_init);