forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-11 6778948f9de86c3cfaf36725a7c87dcff9ba247f 
 kernel/drivers/gpu/drm/nouveau/nouveau_drm.c
....@@ -28,9 +28,11 @@
2828 #include <linux/pci.h>
2929 #include <linux/pm_runtime.h>
3030 #include <linux/vga_switcheroo.h>
31
+#include <linux/mmu_notifier.h>
3132 
32
-#include <drm/drmP.h>
3333 #include <drm/drm_crtc_helper.h>
34
+#include <drm/drm_ioctl.h>
35
+#include <drm/drm_vblank.h>
3436 
3537 #include <core/gpuobj.h>
3638 #include <core/option.h>
....@@ -39,12 +41,12 @@
3941 
4042 #include <nvif/driver.h>
4143 #include <nvif/fifo.h>
44
+#include <nvif/push006c.h>
4245 #include <nvif/user.h>
4346 
4447 #include <nvif/class.h>
4548 #include <nvif/cl0002.h>
4649 #include <nvif/cla06f.h>
47
-#include <nvif/if0004.h>
4850 
4951 #include "nouveau_drv.h"
5052 #include "nouveau_dma.h"
....@@ -63,6 +65,8 @@
6365 #include "nouveau_usif.h"
6466 #include "nouveau_connector.h"
6567 #include "nouveau_platform.h"
68
+#include "nouveau_svm.h"
69
+#include "nouveau_dmem.h"
6670 
6771 MODULE_PARM_DESC(config, "option string to pass to driver core");
6872 static char *nouveau_config;
....@@ -173,11 +177,12 @@
173177 	WARN_ON(!list_empty(&cli->worker));
174178 
175179 	usif_client_fini(cli);
180
+	nouveau_vmm_fini(&cli->svm);
176181 	nouveau_vmm_fini(&cli->vmm);
177
-	nvif_mmu_fini(&cli->mmu);
178
-	nvif_device_fini(&cli->device);
182
+	nvif_mmu_dtor(&cli->mmu);
183
+	nvif_device_dtor(&cli->device);
179184 	mutex_lock(&cli->drm->master.lock);
180
-	nvif_client_fini(&cli->base);
185
+	nvif_client_dtor(&cli->base);
181186 	mutex_unlock(&cli->drm->master.lock);
182187 }
183188 
....@@ -225,7 +230,7 @@
225230 				       cli->name, device, &cli->base);
226231 	} else {
227232 		mutex_lock(&drm->master.lock);
228
-		ret = nvif_client_init(&drm->master.base, cli->name, device,
233
+		ret = nvif_client_ctor(&drm->master.base, cli->name, device,
229234 				       &cli->base);
230235 		mutex_unlock(&drm->master.lock);
231236 	}
....@@ -234,7 +239,7 @@
234239 		goto done;
235240 	}
236241 
237
-	ret = nvif_device_init(&cli->base.object, 0, NV_DEVICE,
242
+	ret = nvif_device_ctor(&cli->base.object, "drmDevice", 0, NV_DEVICE,
238243 			       &(struct nv_device_v0) {
239244 					.device = ~0,
240245 			       }, sizeof(struct nv_device_v0),
....@@ -250,7 +255,8 @@
250255 		goto done;
251256 	}
252257 
253
-	ret = nvif_mmu_init(&cli->device.object, mmus[ret].oclass, &cli->mmu);
258
+	ret = nvif_mmu_ctor(&cli->device.object, "drmMmu", mmus[ret].oclass,
259
+			    &cli->mmu);
254260 	if (ret) {
255261 		NV_PRINTK(err, cli, "MMU allocation failed: %d\n", ret);
256262 		goto done;
....@@ -283,19 +289,132 @@
283289 }
284290 
285291 static void
286
-nouveau_accel_fini(struct nouveau_drm *drm)
292
+nouveau_accel_ce_fini(struct nouveau_drm *drm)
293
+{
294
+	nouveau_channel_idle(drm->cechan);
295
+	nvif_object_dtor(&drm->ttm.copy);
296
+	nouveau_channel_del(&drm->cechan);
297
+}
298
+
299
+static void
300
+nouveau_accel_ce_init(struct nouveau_drm *drm)
301
+{
302
+	struct nvif_device *device = &drm->client.device;
303
+	int ret = 0;
304
+
305
+	/* Allocate channel that has access to a (preferably async) copy
306
+	 * engine, to use for TTM buffer moves.
307
+	 */
308
+	if (device->info.family >= NV_DEVICE_INFO_V0_KEPLER) {
309
+		ret = nouveau_channel_new(drm, device,
310
+					  nvif_fifo_runlist_ce(device), 0,
311
+					  true, &drm->cechan);
312
+	} else
313
+	if (device->info.chipset >= 0xa3 &&
314
+	    device->info.chipset != 0xaa &&
315
+	    device->info.chipset != 0xac) {
316
+		/* Prior to Kepler, there's only a single runlist, so all
317
+		 * engines can be accessed from any channel.
318
+		 *
319
+		 * We still want to use a separate channel though.
320
+		 */
321
+		ret = nouveau_channel_new(drm, device, NvDmaFB, NvDmaTT, false,
322
+					  &drm->cechan);
323
+	}
324
+
325
+	if (ret)
326
+		NV_ERROR(drm, "failed to create ce channel, %d\n", ret);
327
+}
328
+
329
+static void
330
+nouveau_accel_gr_fini(struct nouveau_drm *drm)
287331 {
288332 	nouveau_channel_idle(drm->channel);
289
-	nvif_object_fini(&drm->ntfy);
333
+	nvif_object_dtor(&drm->ntfy);
290334 	nvkm_gpuobj_del(&drm->notify);
291
-	nvif_notify_fini(&drm->flip);
292
-	nvif_object_fini(&drm->nvsw);
293335 	nouveau_channel_del(&drm->channel);
336
+}
294337 
295
-	nouveau_channel_idle(drm->cechan);
296
-	nvif_object_fini(&drm->ttm.copy);
297
-	nouveau_channel_del(&drm->cechan);
338
+static void
339
+nouveau_accel_gr_init(struct nouveau_drm *drm)
340
+{
341
+	struct nvif_device *device = &drm->client.device;
342
+	u32 arg0, arg1;
343
+	int ret;
298344 
345
+	/* Allocate channel that has access to the graphics engine. */
346
+	if (device->info.family >= NV_DEVICE_INFO_V0_KEPLER) {
347
+		arg0 = nvif_fifo_runlist(device, NV_DEVICE_INFO_ENGINE_GR);
348
+		arg1 = 1;
349
+	} else {
350
+		arg0 = NvDmaFB;
351
+		arg1 = NvDmaTT;
352
+	}
353
+
354
+	ret = nouveau_channel_new(drm, device, arg0, arg1, false,
355
+				  &drm->channel);
356
+	if (ret) {
357
+		NV_ERROR(drm, "failed to create kernel channel, %d\n", ret);
358
+		nouveau_accel_gr_fini(drm);
359
+		return;
360
+	}
361
+
362
+	/* A SW class is used on pre-NV50 HW to assist with handling the
363
+	 * synchronisation of page flips, as well as to implement fences
364
+	 * on TNT/TNT2 HW that lacks any kind of support in host.
365
+	 */
366
+	if (!drm->channel->nvsw.client && device->info.family < NV_DEVICE_INFO_V0_TESLA) {
367
+		ret = nvif_object_ctor(&drm->channel->user, "drmNvsw",
368
+				       NVDRM_NVSW, nouveau_abi16_swclass(drm),
369
+				       NULL, 0, &drm->channel->nvsw);
370
+		if (ret == 0) {
371
+			struct nvif_push *push = drm->channel->chan.push;
372
+			ret = PUSH_WAIT(push, 2);
373
+			if (ret == 0)
374
+				PUSH_NVSQ(push, NV_SW, 0x0000, drm->channel->nvsw.handle);
375
+		}
376
+
377
+		if (ret) {
378
+			NV_ERROR(drm, "failed to allocate sw class, %d\n", ret);
379
+			nouveau_accel_gr_fini(drm);
380
+			return;
381
+		}
382
+	}
383
+
384
+	/* NvMemoryToMemoryFormat requires a notifier ctxdma for some reason,
385
+	 * even if notification is never requested, so, allocate a ctxdma on
386
+	 * any GPU where it's possible we'll end up using M2MF for BO moves.
387
+	 */
388
+	if (device->info.family < NV_DEVICE_INFO_V0_FERMI) {
389
+		ret = nvkm_gpuobj_new(nvxx_device(device), 32, 0, false, NULL,
390
+				      &drm->notify);
391
+		if (ret) {
392
+			NV_ERROR(drm, "failed to allocate notifier, %d\n", ret);
393
+			nouveau_accel_gr_fini(drm);
394
+			return;
395
+		}
396
+
397
+		ret = nvif_object_ctor(&drm->channel->user, "drmM2mfNtfy",
398
+				       NvNotify0, NV_DMA_IN_MEMORY,
399
+				       &(struct nv_dma_v0) {
400
+						.target = NV_DMA_V0_TARGET_VRAM,
401
+						.access = NV_DMA_V0_ACCESS_RDWR,
402
+						.start = drm->notify->addr,
403
+						.limit = drm->notify->addr + 31
404
+				       }, sizeof(struct nv_dma_v0),
405
+				       &drm->ntfy);
406
+		if (ret) {
407
+			nouveau_accel_gr_fini(drm);
408
+			return;
409
+		}
410
+	}
411
+}
412
+
413
+static void
414
+nouveau_accel_fini(struct nouveau_drm *drm)
415
+{
416
+	nouveau_accel_ce_fini(drm);
417
+	nouveau_accel_gr_fini(drm);
299418 	if (drm->fence)
300419 		nouveau_fence(drm)->dtor(drm);
301420 }
....@@ -305,23 +424,16 @@
305424 {
306425 	struct nvif_device *device = &drm->client.device;
307426 	struct nvif_sclass *sclass;
308
-	u32 arg0, arg1;
309427 	int ret, i, n;
310428 
311429 	if (nouveau_noaccel)
312430 		return;
313431 
432
+	/* Initialise global support for channels, and synchronisation. */
314433 	ret = nouveau_channels_init(drm);
315434 	if (ret)
316435 		return;
317436 
318
-	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_VOLTA) {
319
-		ret = nvif_user_init(device);
320
-		if (ret)
321
-			return;
322
-	}
323
-
324
-	/* initialise synchronisation routines */
325437 	/*XXX: this is crap, but the fence/channel stuff is a little
326438 	 *     backwards in some places.  this will be fixed.
327439 	 */
....@@ -353,6 +465,7 @@
353465 		case MAXWELL_CHANNEL_GPFIFO_A:
354466 		case PASCAL_CHANNEL_GPFIFO_A:
355467 		case VOLTA_CHANNEL_GPFIFO_A:
468
+		case TURING_CHANNEL_GPFIFO_A:
356469 			ret = nvc0_fence_create(drm);
357470 			break;
358471 		default:
....@@ -367,166 +480,57 @@
367480 		return;
368481 	}
369482 
370
-	if (device->info.family >= NV_DEVICE_INFO_V0_KEPLER) {
371
-		ret = nouveau_channel_new(drm, &drm->client.device,
372
-					  nvif_fifo_runlist_ce(device), 0,
373
-					  &drm->cechan);
483
+	/* Volta requires access to a doorbell register for kickoff. */
484
+	if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_VOLTA) {
485
+		ret = nvif_user_ctor(device, "drmUsermode");
374486 		if (ret)
375
-			NV_ERROR(drm, "failed to create ce channel, %d\n", ret);
376
-
377
-		arg0 = nvif_fifo_runlist(device, NV_DEVICE_INFO_ENGINE_GR);
378
-		arg1 = 1;
379
-	} else
380
-	if (device->info.chipset >= 0xa3 &&
381
-	    device->info.chipset != 0xaa &&
382
-	    device->info.chipset != 0xac) {
383
-		ret = nouveau_channel_new(drm, &drm->client.device,
384
-					  NvDmaFB, NvDmaTT, &drm->cechan);
385
-		if (ret)
386
-			NV_ERROR(drm, "failed to create ce channel, %d\n", ret);
387
-
388
-		arg0 = NvDmaFB;
389
-		arg1 = NvDmaTT;
390
-	} else {
391
-		arg0 = NvDmaFB;
392
-		arg1 = NvDmaTT;
393
-	}
394
-
395
-	ret = nouveau_channel_new(drm, &drm->client.device,
396
-				  arg0, arg1, &drm->channel);
397
-	if (ret) {
398
-		NV_ERROR(drm, "failed to create kernel channel, %d\n", ret);
399
-		nouveau_accel_fini(drm);
400
-		return;
401
-	}
402
-
403
-	if (device->info.family < NV_DEVICE_INFO_V0_TESLA) {
404
-		ret = nvif_object_init(&drm->channel->user, NVDRM_NVSW,
405
-				       nouveau_abi16_swclass(drm), NULL, 0,
406
-				       &drm->nvsw);
407
-		if (ret == 0) {
408
-			ret = RING_SPACE(drm->channel, 2);
409
-			if (ret == 0) {
410
-				BEGIN_NV04(drm->channel, NvSubSw, 0, 1);
411
-				OUT_RING  (drm->channel, drm->nvsw.handle);
412
-			}
413
-
414
-			ret = nvif_notify_init(&drm->nvsw,
415
-					       nouveau_flip_complete,
416
-					       false, NV04_NVSW_NTFY_UEVENT,
417
-					       NULL, 0, 0, &drm->flip);
418
-			if (ret == 0)
419
-				ret = nvif_notify_get(&drm->flip);
420
-			if (ret) {
421
-				nouveau_accel_fini(drm);
422
-				return;
423
-			}
424
-		}
425
-
426
-		if (ret) {
427
-			NV_ERROR(drm, "failed to allocate sw class, %d\n", ret);
428
-			nouveau_accel_fini(drm);
429487 			return;
430
-		}
431488 	}
432489 
433
-	if (device->info.family < NV_DEVICE_INFO_V0_FERMI) {
434
-		ret = nvkm_gpuobj_new(nvxx_device(&drm->client.device), 32, 0,
435
-				      false, NULL, &drm->notify);
436
-		if (ret) {
437
-			NV_ERROR(drm, "failed to allocate notifier, %d\n", ret);
438
-			nouveau_accel_fini(drm);
439
-			return;
440
-		}
490
+	/* Allocate channels we need to support various functions. */
491
+	nouveau_accel_gr_init(drm);
492
+	nouveau_accel_ce_init(drm);
441493 
442
-		ret = nvif_object_init(&drm->channel->user, NvNotify0,
443
-				       NV_DMA_IN_MEMORY,
444
-				       &(struct nv_dma_v0) {
445
-						.target = NV_DMA_V0_TARGET_VRAM,
446
-						.access = NV_DMA_V0_ACCESS_RDWR,
447
-						.start = drm->notify->addr,
448
-						.limit = drm->notify->addr + 31
449
-				       }, sizeof(struct nv_dma_v0),
450
-				       &drm->ntfy);
451
-		if (ret) {
452
-			nouveau_accel_fini(drm);
453
-			return;
454
-		}
455
-	}
456
-
457
-
494
+	/* Initialise accelerated TTM buffer moves. */
458495 	nouveau_bo_move_init(drm);
459496 }
460497 
461
-static int nouveau_drm_probe(struct pci_dev *pdev,
462
-			     const struct pci_device_id *pent)
498
+static void __printf(2, 3)
499
+nouveau_drm_errorf(struct nvif_object *object, const char *fmt, ...)
463500 {
464
-	struct nvkm_device *device;
465
-	struct apertures_struct *aper;
466
-	bool boot = false;
467
-	int ret;
501
+	struct nouveau_drm *drm = container_of(object->parent, typeof(*drm), parent);
502
+	struct va_format vaf;
503
+	va_list va;
468504 
469
-	if (vga_switcheroo_client_probe_defer(pdev))
470
-		return -EPROBE_DEFER;
471
-
472
-	/* We need to check that the chipset is supported before booting
473
-	 * fbdev off the hardware, as there's no way to put it back.
474
-	 */
475
-	ret = nvkm_device_pci_new(pdev, NULL, "error", true, false, 0, &device);
476
-	if (ret)
477
-		return ret;
478
-
479
-	nvkm_device_del(&device);
480
-
481
-	/* Remove conflicting drivers (vesafb, efifb etc). */
482
-	aper = alloc_apertures(3);
483
-	if (!aper)
484
-		return -ENOMEM;
485
-
486
-	aper->ranges[0].base = pci_resource_start(pdev, 1);
487
-	aper->ranges[0].size = pci_resource_len(pdev, 1);
488
-	aper->count = 1;
489
-
490
-	if (pci_resource_len(pdev, 2)) {
491
-		aper->ranges[aper->count].base = pci_resource_start(pdev, 2);
492
-		aper->ranges[aper->count].size = pci_resource_len(pdev, 2);
493
-		aper->count++;
494
-	}
495
-
496
-	if (pci_resource_len(pdev, 3)) {
497
-		aper->ranges[aper->count].base = pci_resource_start(pdev, 3);
498
-		aper->ranges[aper->count].size = pci_resource_len(pdev, 3);
499
-		aper->count++;
500
-	}
501
-
502
-#ifdef CONFIG_X86
503
-	boot = pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
504
-#endif
505
-	if (nouveau_modeset != 2)
506
-		drm_fb_helper_remove_conflicting_framebuffers(aper, "nouveaufb", boot);
507
-	kfree(aper);
508
-
509
-	ret = nvkm_device_pci_new(pdev, nouveau_config, nouveau_debug,
510
-				  true, true, ~0ULL, &device);
511
-	if (ret)
512
-		return ret;
513
-
514
-	pci_set_master(pdev);
515
-
516
-	if (nouveau_atomic)
517
-		driver_pci.driver_features |= DRIVER_ATOMIC;
518
-
519
-	ret = drm_get_pci_dev(pdev, pent, &driver_pci);
520
-	if (ret) {
521
-		nvkm_device_del(&device);
522
-		return ret;
523
-	}
524
-
525
-	return 0;
505
+	va_start(va, fmt);
506
+	vaf.fmt = fmt;
507
+	vaf.va = &va;
508
+	NV_ERROR(drm, "%pV", &vaf);
509
+	va_end(va);
526510 }
527511 
512
+static void __printf(2, 3)
513
+nouveau_drm_debugf(struct nvif_object *object, const char *fmt, ...)
514
+{
515
+	struct nouveau_drm *drm = container_of(object->parent, typeof(*drm), parent);
516
+	struct va_format vaf;
517
+	va_list va;
518
+
519
+	va_start(va, fmt);
520
+	vaf.fmt = fmt;
521
+	vaf.va = &va;
522
+	NV_DEBUG(drm, "%pV", &vaf);
523
+	va_end(va);
524
+}
525
+
526
+static const struct nvif_parent_func
527
+nouveau_parent = {
528
+	.debugf = nouveau_drm_debugf,
529
+	.errorf = nouveau_drm_errorf,
530
+};
531
+
528532 static int
529
-nouveau_drm_load(struct drm_device *dev, unsigned long flags)
533
+nouveau_drm_device_init(struct drm_device *dev)
530534 {
531535 	struct nouveau_drm *drm;
532536 	int ret;
....@@ -536,13 +540,16 @@
536540 	dev->dev_private = drm;
537541 	drm->dev = dev;
538542 
543
+	nvif_parent_ctor(&nouveau_parent, &drm->parent);
544
+	drm->master.base.object.parent = &drm->parent;
545
+
539546 	ret = nouveau_cli_init(drm, "DRM-master", &drm->master);
540547 	if (ret)
541
-		return ret;
548
+		goto fail_alloc;
542549 
543550 	ret = nouveau_cli_init(drm, "DRM", &drm->client);
544551 	if (ret)
545
-		return ret;
552
+		goto fail_master;
546553 
547554 	dev->irq_enabled = true;
548555 
....@@ -550,6 +557,7 @@
550557 		nvkm_dbgopt(nouveau_debug, "DRM");
551558 
552559 	INIT_LIST_HEAD(&drm->clients);
560
+	mutex_init(&drm->clients_lock);
553561 	spin_lock_init(&drm->tile.lock);
554562 
555563 	/* workaround an odd issue on nvc1 by disabling the device's
....@@ -569,19 +577,22 @@
569577 	if (ret)
570578 		goto fail_bios;
571579 
580
+	nouveau_accel_init(drm);
581
+
572582 	ret = nouveau_display_create(dev);
573583 	if (ret)
574584 		goto fail_dispctor;
575585 
576586 	if (dev->mode_config.num_crtc) {
577
-		ret = nouveau_display_init(dev);
587
+		ret = nouveau_display_init(dev, false, false);
578588 		if (ret)
579589 			goto fail_dispinit;
580590 	}
581591 
582592 	nouveau_debugfs_init(drm);
583593 	nouveau_hwmon_init(dev);
584
-	nouveau_accel_init(drm);
594
+	nouveau_svm_init(drm);
595
+	nouveau_dmem_init(drm);
585596 	nouveau_fbcon_init(dev);
586597 	nouveau_led_init(dev);
587598 
....@@ -599,20 +610,25 @@
599610 fail_dispinit:
600611 	nouveau_display_destroy(dev);
601612 fail_dispctor:
613
+	nouveau_accel_fini(drm);
602614 	nouveau_bios_takedown(dev);
603615 fail_bios:
604616 	nouveau_ttm_fini(drm);
605617 fail_ttm:
606618 	nouveau_vga_fini(drm);
607619 	nouveau_cli_fini(&drm->client);
620
+fail_master:
608621 	nouveau_cli_fini(&drm->master);
622
+fail_alloc:
623
+	nvif_parent_dtor(&drm->parent);
609624 	kfree(drm);
610625 	return ret;
611626 }
612627 
613628 static void
614
-nouveau_drm_unload(struct drm_device *dev)
629
+nouveau_drm_device_fini(struct drm_device *dev)
615630 {
631
+	struct nouveau_cli *cli, *temp_cli;
616632 	struct nouveau_drm *drm = nouveau_drm(dev);
617633 
618634 	if (nouveau_pmops_runtime()) {
....@@ -622,7 +638,8 @@
622638 
623639 	nouveau_led_fini(dev);
624640 	nouveau_fbcon_fini(dev);
625
-	nouveau_accel_fini(drm);
641
+	nouveau_dmem_fini(drm);
642
+	nouveau_svm_fini(drm);
626643 	nouveau_hwmon_fini(dev);
627644 	nouveau_debugfs_fini(drm);
628645 
....@@ -630,14 +647,163 @@
630647 		nouveau_display_fini(dev, false, false);
631648 	nouveau_display_destroy(dev);
632649 
650
+	nouveau_accel_fini(drm);
633651 	nouveau_bios_takedown(dev);
634652 
635653 	nouveau_ttm_fini(drm);
636654 	nouveau_vga_fini(drm);
637655 
656
+	/*
657
+	 * There may be existing clients from as-yet unclosed files. For now,
658
+	 * clean them up here rather than deferring until the file is closed,
659
+	 * but this likely not correct if we want to support hot-unplugging
660
+	 * properly.
661
+	 */
662
+	mutex_lock(&drm->clients_lock);
663
+	list_for_each_entry_safe(cli, temp_cli, &drm->clients, head) {
664
+		list_del(&cli->head);
665
+		mutex_lock(&cli->mutex);
666
+		if (cli->abi16)
667
+			nouveau_abi16_fini(cli->abi16);
668
+		mutex_unlock(&cli->mutex);
669
+		nouveau_cli_fini(cli);
670
+		kfree(cli);
671
+	}
672
+	mutex_unlock(&drm->clients_lock);
673
+
638674 	nouveau_cli_fini(&drm->client);
639675 	nouveau_cli_fini(&drm->master);
676
+	nvif_parent_dtor(&drm->parent);
677
+	mutex_destroy(&drm->clients_lock);
640678 	kfree(drm);
679
+}
680
+
681
+/*
682
+ * On some Intel PCIe bridge controllers doing a
683
+ * D0 -> D3hot -> D3cold -> D0 sequence causes Nvidia GPUs to not reappear.
684
+ * Skipping the intermediate D3hot step seems to make it work again. This is
685
+ * probably caused by not meeting the expectation the involved AML code has
686
+ * when the GPU is put into D3hot state before invoking it.
687
+ *
688
+ * This leads to various manifestations of this issue:
689
+ *  - AML code execution to power on the GPU hits an infinite loop (as the
690
+ *    code waits on device memory to change).
691
+ *  - kernel crashes, as all PCI reads return -1, which most code isn't able
692
+ *    to handle well enough.
693
+ *
694
+ * In all cases dmesg will contain at least one line like this:
695
+ * 'nouveau 0000:01:00.0: Refused to change power state, currently in D3'
696
+ * followed by a lot of nouveau timeouts.
697
+ *
698
+ * In the \_SB.PCI0.PEG0.PG00._OFF code deeper down writes bit 0x80 to the not
699
+ * documented PCI config space register 0x248 of the Intel PCIe bridge
700
+ * controller (0x1901) in order to change the state of the PCIe link between
701
+ * the PCIe port and the GPU. There are alternative code paths using other
702
+ * registers, which seem to work fine (executed pre Windows 8):
703
+ *  - 0xbc bit 0x20 (publicly available documentation claims 'reserved')
704
+ *  - 0xb0 bit 0x10 (link disable)
705
+ * Changing the conditions inside the firmware by poking into the relevant
706
+ * addresses does resolve the issue, but it seemed to be ACPI private memory
707
+ * and not any device accessible memory at all, so there is no portable way of
708
+ * changing the conditions.
709
+ * On a XPS 9560 that means bits [0,3] on \CPEX need to be cleared.
710
+ *
711
+ * The only systems where this behavior can be seen are hybrid graphics laptops
712
+ * with a secondary Nvidia Maxwell, Pascal or Turing GPU. It's unclear whether
713
+ * this issue only occurs in combination with listed Intel PCIe bridge
714
+ * controllers and the mentioned GPUs or other devices as well.
715
+ *
716
+ * documentation on the PCIe bridge controller can be found in the
717
+ * "7th Generation Intel® Processor Families for H Platforms Datasheet Volume 2"
718
+ * Section "12 PCI Express* Controller (x16) Registers"
719
+ */
720
+
721
+static void quirk_broken_nv_runpm(struct pci_dev *pdev)
722
+{
723
+	struct drm_device *dev = pci_get_drvdata(pdev);
724
+	struct nouveau_drm *drm = nouveau_drm(dev);
725
+	struct pci_dev *bridge = pci_upstream_bridge(pdev);
726
+
727
+	if (!bridge || bridge->vendor != PCI_VENDOR_ID_INTEL)
728
+		return;
729
+
730
+	switch (bridge->device) {
731
+	case 0x1901:
732
+		drm->old_pm_cap = pdev->pm_cap;
733
+		pdev->pm_cap = 0;
734
+		NV_INFO(drm, "Disabling PCI power management to avoid bug\n");
735
+		break;
736
+	}
737
+}
738
+
739
+static int nouveau_drm_probe(struct pci_dev *pdev,
740
+			     const struct pci_device_id *pent)
741
+{
742
+	struct nvkm_device *device;
743
+	struct drm_device *drm_dev;
744
+	int ret;
745
+
746
+	if (vga_switcheroo_client_probe_defer(pdev))
747
+		return -EPROBE_DEFER;
748
+
749
+	/* We need to check that the chipset is supported before booting
750
+	 * fbdev off the hardware, as there's no way to put it back.
751
+	 */
752
+	ret = nvkm_device_pci_new(pdev, nouveau_config, "error",
753
+				  true, false, 0, &device);
754
+	if (ret)
755
+		return ret;
756
+
757
+	nvkm_device_del(&device);
758
+
759
+	/* Remove conflicting drivers (vesafb, efifb etc). */
760
+	ret = drm_fb_helper_remove_conflicting_pci_framebuffers(pdev, "nouveaufb");
761
+	if (ret)
762
+		return ret;
763
+
764
+	ret = nvkm_device_pci_new(pdev, nouveau_config, nouveau_debug,
765
+				  true, true, ~0ULL, &device);
766
+	if (ret)
767
+		return ret;
768
+
769
+	pci_set_master(pdev);
770
+
771
+	if (nouveau_atomic)
772
+		driver_pci.driver_features |= DRIVER_ATOMIC;
773
+
774
+	drm_dev = drm_dev_alloc(&driver_pci, &pdev->dev);
775
+	if (IS_ERR(drm_dev)) {
776
+		ret = PTR_ERR(drm_dev);
777
+		goto fail_nvkm;
778
+	}
779
+
780
+	ret = pci_enable_device(pdev);
781
+	if (ret)
782
+		goto fail_drm;
783
+
784
+	drm_dev->pdev = pdev;
785
+	pci_set_drvdata(pdev, drm_dev);
786
+
787
+	ret = nouveau_drm_device_init(drm_dev);
788
+	if (ret)
789
+		goto fail_pci;
790
+
791
+	ret = drm_dev_register(drm_dev, pent->driver_data);
792
+	if (ret)
793
+		goto fail_drm_dev_init;
794
+
795
+	quirk_broken_nv_runpm(pdev);
796
+	return 0;
797
+
798
+fail_drm_dev_init:
799
+	nouveau_drm_device_fini(drm_dev);
800
+fail_pci:
801
+	pci_disable_device(pdev);
802
+fail_drm:
803
+	drm_dev_put(drm_dev);
804
+fail_nvkm:
805
+	nvkm_device_del(&device);
806
+	return ret;
641807 }
642808 
643809 void
....@@ -647,11 +813,14 @@
647813 	struct nvkm_client *client;
648814 	struct nvkm_device *device;
649815 
816
+	drm_dev_unplug(dev);
817
+
650818 	dev->irq_enabled = false;
651819 	client = nvxx_client(&drm->client.base);
652820 	device = nvkm_device_find(client->device);
653
-	drm_put_dev(dev);
654821 
822
+	nouveau_drm_device_fini(dev);
823
+	drm_dev_put(dev);
655824 	nvkm_device_del(&device);
656825 }
657826 
....@@ -659,8 +828,13 @@
659828 nouveau_drm_remove(struct pci_dev *pdev)
660829 {
661830 	struct drm_device *dev = pci_get_drvdata(pdev);
831
+	struct nouveau_drm *drm = nouveau_drm(dev);
662832 
833
+	/* revert our workaround */
834
+	if (drm->old_pm_cap)
835
+		pdev->pm_cap = drm->old_pm_cap;
663836 	nouveau_drm_device_remove(dev);
837
+	pci_disable_device(pdev);
664838 }
665839 
666840 static int
....@@ -669,6 +843,8 @@
669843 	struct nouveau_drm *drm = nouveau_drm(dev);
670844 	int ret;
671845 
846
+	nouveau_svm_suspend(drm);
847
+	nouveau_dmem_suspend(drm);
672848 	nouveau_led_suspend(dev);
673849 
674850 	if (dev->mode_config.num_crtc) {
....@@ -726,10 +902,15 @@
726902 static int
727903 nouveau_do_resume(struct drm_device *dev, bool runtime)
728904 {
905
+	int ret = 0;
729906 	struct nouveau_drm *drm = nouveau_drm(dev);
730907 
731908 	NV_DEBUG(drm, "resuming object tree...\n");
732
-	nvif_client_resume(&drm->master.base);
909
+	ret = nvif_client_resume(&drm->master.base);
910
+	if (ret) {
911
+		NV_ERROR(drm, "Client resume failed with error: %d\n", ret);
912
+		return ret;
913
+	}
733914 
734915 	NV_DEBUG(drm, "resuming fence...\n");
735916 	if (drm->fence && nouveau_fence(drm)->resume)
....@@ -745,7 +926,8 @@
745926 	}
746927 
747928 	nouveau_led_resume(dev);
748
-
929
+	nouveau_dmem_resume(drm);
930
+	nouveau_svm_resume(drm);
749931 	return 0;
750932 }
751933 
....@@ -792,7 +974,7 @@
792974 	ret = nouveau_do_resume(drm_dev, false);
793975 
794976 	/* Monitors may have been connected / disconnected during suspend */
795
-	schedule_work(&nouveau_drm(drm_dev)->hpd_work);
977
+	nouveau_display_hpd_resume(drm_dev);
796978 
797979 	return ret;
798980 }
....@@ -848,6 +1030,7 @@
8481030 {
8491031 	struct pci_dev *pdev = to_pci_dev(dev);
8501032 	struct drm_device *drm_dev = pci_get_drvdata(pdev);
1033
+	struct nouveau_drm *drm = nouveau_drm(drm_dev);
8511034 	struct nvif_device *device = &nouveau_drm(drm_dev)->client.device;
8521035 	int ret;
8531036 
....@@ -864,13 +1047,17 @@
8641047 	pci_set_master(pdev);
8651048 
8661049 	ret = nouveau_do_resume(drm_dev, true);
1050
+	if (ret) {
1051
+		NV_ERROR(drm, "resume failed with: %d\n", ret);
1052
+		return ret;
1053
+	}
8671054 
8681055 	/* do magic */
8691056 	nvif_mask(&device->object, 0x088488, (1 << 25), (1 << 25));
8701057 	drm_dev->switch_power_state = DRM_SWITCH_POWER_ON;
8711058 
8721059 	/* Monitors may have been connected / disconnected during suspend */
873
-	schedule_work(&nouveau_drm(drm_dev)->hpd_work);
1060
+	nouveau_display_hpd_resume(drm_dev);
8741061 
8751062 	return ret;
8761063 }
....@@ -920,9 +1107,9 @@
9201107 
9211108 	fpriv->driver_priv = cli;
9221109 
923
-	mutex_lock(&drm->client.mutex);
1110
+	mutex_lock(&drm->clients_lock);
9241111 	list_add(&cli->head, &drm->clients);
925
-	mutex_unlock(&drm->client.mutex);
1112
+	mutex_unlock(&drm->clients_lock);
9261113 
9271114 done:
9281115 	if (ret && cli) {
....@@ -940,6 +1127,16 @@
9401127 {
9411128 	struct nouveau_cli *cli = nouveau_cli(fpriv);
9421129 	struct nouveau_drm *drm = nouveau_drm(dev);
1130
+	int dev_index;
1131
+
1132
+	/*
1133
+	 * The device is gone, and as it currently stands all clients are
1134
+	 * cleaned up in the removal codepath. In the future this may change
1135
+	 * so that we can support hot-unplugging, but for now we immediately
1136
+	 * return to avoid a double-free situation.
1137
+	 */
1138
+	if (!drm_dev_enter(dev, &dev_index))
1139
+		return;
9431140 
9441141 	pm_runtime_get_sync(dev->dev);
9451142 
....@@ -948,30 +1145,33 @@
9481145 		nouveau_abi16_fini(cli->abi16);
9491146 	mutex_unlock(&cli->mutex);
9501147 
951
-	mutex_lock(&drm->client.mutex);
1148
+	mutex_lock(&drm->clients_lock);
9521149 	list_del(&cli->head);
953
-	mutex_unlock(&drm->client.mutex);
1150
+	mutex_unlock(&drm->clients_lock);
9541151 
9551152 	nouveau_cli_fini(cli);
9561153 	kfree(cli);
9571154 	pm_runtime_mark_last_busy(dev->dev);
9581155 	pm_runtime_put_autosuspend(dev->dev);
1156
+	drm_dev_exit(dev_index);
9591157 }
9601158 
9611159 static const struct drm_ioctl_desc
9621160 nouveau_ioctls[] = {
963
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GETPARAM, nouveau_abi16_ioctl_getparam, DRM_AUTH|DRM_RENDER_ALLOW),
964
-	DRM_IOCTL_DEF_DRV(NOUVEAU_SETPARAM, nouveau_abi16_ioctl_setparam, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
965
-	DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_ALLOC, nouveau_abi16_ioctl_channel_alloc, DRM_AUTH|DRM_RENDER_ALLOW),
966
-	DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_FREE, nouveau_abi16_ioctl_channel_free, DRM_AUTH|DRM_RENDER_ALLOW),
967
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GROBJ_ALLOC, nouveau_abi16_ioctl_grobj_alloc, DRM_AUTH|DRM_RENDER_ALLOW),
968
-	DRM_IOCTL_DEF_DRV(NOUVEAU_NOTIFIEROBJ_ALLOC, nouveau_abi16_ioctl_notifierobj_alloc, DRM_AUTH|DRM_RENDER_ALLOW),
969
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GPUOBJ_FREE, nouveau_abi16_ioctl_gpuobj_free, DRM_AUTH|DRM_RENDER_ALLOW),
970
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_NEW, nouveau_gem_ioctl_new, DRM_AUTH|DRM_RENDER_ALLOW),
971
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_PUSHBUF, nouveau_gem_ioctl_pushbuf, DRM_AUTH|DRM_RENDER_ALLOW),
972
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_PREP, nouveau_gem_ioctl_cpu_prep, DRM_AUTH|DRM_RENDER_ALLOW),
973
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_FINI, nouveau_gem_ioctl_cpu_fini, DRM_AUTH|DRM_RENDER_ALLOW),
974
-	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_INFO, nouveau_gem_ioctl_info, DRM_AUTH|DRM_RENDER_ALLOW),
1161
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GETPARAM, nouveau_abi16_ioctl_getparam, DRM_RENDER_ALLOW),
1162
+	DRM_IOCTL_DEF_DRV(NOUVEAU_SETPARAM, drm_invalid_op, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
1163
+	DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_ALLOC, nouveau_abi16_ioctl_channel_alloc, DRM_RENDER_ALLOW),
1164
+	DRM_IOCTL_DEF_DRV(NOUVEAU_CHANNEL_FREE, nouveau_abi16_ioctl_channel_free, DRM_RENDER_ALLOW),
1165
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GROBJ_ALLOC, nouveau_abi16_ioctl_grobj_alloc, DRM_RENDER_ALLOW),
1166
+	DRM_IOCTL_DEF_DRV(NOUVEAU_NOTIFIEROBJ_ALLOC, nouveau_abi16_ioctl_notifierobj_alloc, DRM_RENDER_ALLOW),
1167
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GPUOBJ_FREE, nouveau_abi16_ioctl_gpuobj_free, DRM_RENDER_ALLOW),
1168
+	DRM_IOCTL_DEF_DRV(NOUVEAU_SVM_INIT, nouveau_svmm_init, DRM_RENDER_ALLOW),
1169
+	DRM_IOCTL_DEF_DRV(NOUVEAU_SVM_BIND, nouveau_svmm_bind, DRM_RENDER_ALLOW),
1170
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_NEW, nouveau_gem_ioctl_new, DRM_RENDER_ALLOW),
1171
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_PUSHBUF, nouveau_gem_ioctl_pushbuf, DRM_RENDER_ALLOW),
1172
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_PREP, nouveau_gem_ioctl_cpu_prep, DRM_RENDER_ALLOW),
1173
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_CPU_FINI, nouveau_gem_ioctl_cpu_fini, DRM_RENDER_ALLOW),
1174
+	DRM_IOCTL_DEF_DRV(NOUVEAU_GEM_INFO, nouveau_gem_ioctl_info, DRM_RENDER_ALLOW),
9751175 };
9761176 
9771177 long
....@@ -1019,14 +1219,12 @@
10191219 static struct drm_driver
10201220 driver_stub = {
10211221 	.driver_features =
1022
-		DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME | DRIVER_RENDER
1222
+		DRIVER_GEM | DRIVER_MODESET | DRIVER_RENDER
10231223 #if defined(CONFIG_NOUVEAU_LEGACY_CTX_SUPPORT)
10241224 		| DRIVER_KMS_LEGACY_CONTEXT
10251225 #endif
10261226 		,
10271227 
1028
-	.load = nouveau_drm_load,
1029
-	.unload = nouveau_drm_unload,
10301228 	.open = nouveau_drm_open,
10311229 	.postclose = nouveau_drm_postclose,
10321230 	.lastclose = nouveau_vga_lastclose,
....@@ -1035,21 +1233,13 @@
10351233 	.debugfs_init = nouveau_drm_debugfs_init,
10361234 #endif
10371235 
1038
-	.enable_vblank = nouveau_display_vblank_enable,
1039
-	.disable_vblank = nouveau_display_vblank_disable,
1040
-	.get_scanout_position = nouveau_display_scanoutpos,
1041
-	.get_vblank_timestamp = drm_calc_vbltimestamp_from_scanoutpos,
1042
-
10431236 	.ioctls = nouveau_ioctls,
10441237 	.num_ioctls = ARRAY_SIZE(nouveau_ioctls),
10451238 	.fops = &nouveau_driver_fops,
10461239 
10471240 	.prime_handle_to_fd = drm_gem_prime_handle_to_fd,
10481241 	.prime_fd_to_handle = drm_gem_prime_fd_to_handle,
1049
-	.gem_prime_export = drm_gem_prime_export,
1050
-	.gem_prime_import = drm_gem_prime_import,
10511242 	.gem_prime_pin = nouveau_gem_prime_pin,
1052
-	.gem_prime_res_obj = nouveau_gem_prime_res_obj,
10531243 	.gem_prime_unpin = nouveau_gem_prime_unpin,
10541244 	.gem_prime_get_sg_table = nouveau_gem_prime_get_sg_table,
10551245 	.gem_prime_import_sg_table = nouveau_gem_prime_import_sg_table,
....@@ -1147,10 +1337,16 @@
11471337 		goto err_free;
11481338 	}
11491339 
1340
+	err = nouveau_drm_device_init(drm);
1341
+	if (err)
1342
+		goto err_put;
1343
+
11501344 	platform_set_drvdata(pdev, drm);
11511345 
11521346 	return drm;
11531347 
1348
+err_put:
1349
+	drm_dev_put(drm);
11541350 err_free:
11551351 	nvkm_device_del(pdevice);
11561352 
....@@ -1202,6 +1398,8 @@
12021398 #ifdef CONFIG_NOUVEAU_PLATFORM_DRIVER
12031399 	platform_driver_unregister(&nouveau_platform_driver);
12041400 #endif
1401
+	if (IS_ENABLED(CONFIG_DRM_NOUVEAU_SVM))
1402
+		mmu_notifier_synchronize();
12051403 }
12061404 
12071405 module_init(nouveau_drm_init);