change otg to host mode

hc

2024-05-11 04dd17822334871b23ea2862f7798fb0e0007777

 kernel/drivers/gpu/drm/i915/i915_gem.c
..
..
@@ -25,247 +25,83 @@
25
25
  *
26
26
  */
27
27
 
28
-#include <drm/drmP.h>
29
28
 #include <drm/drm_vma_manager.h>
30
-#include <drm/i915_drm.h>
31
-#include "i915_drv.h"
32
-#include "i915_gem_clflush.h"
33
-#include "i915_vgpu.h"
34
-#include "i915_trace.h"
35
-#include "intel_drv.h"
36
-#include "intel_frontbuffer.h"
37
-#include "intel_mocs.h"
38
-#include "intel_workarounds.h"
39
-#include "i915_gemfs.h"
40
29
 #include <linux/dma-fence-array.h>
41
30
 #include <linux/kthread.h>
42
-#include <linux/reservation.h>
31
+#include <linux/dma-resv.h>
43
32
 #include <linux/shmem_fs.h>
44
33
 #include <linux/slab.h>
45
34
 #include <linux/stop_machine.h>
46
35
 #include <linux/swap.h>
47
36
 #include <linux/pci.h>
48
37
 #include <linux/dma-buf.h>
38
+#include <linux/mman.h>
49
39
 
50
-static void i915_gem_flush_free_objects(struct drm_i915_private *i915);
40
+#include "display/intel_display.h"
41
+#include "display/intel_frontbuffer.h"
51
42
 
52
-static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
53
-{
54
-	if (obj->cache_dirty)
55
-		return false;
43
+#include "gem/i915_gem_clflush.h"
44
+#include "gem/i915_gem_context.h"
45
+#include "gem/i915_gem_ioctls.h"
46
+#include "gem/i915_gem_mman.h"
47
+#include "gem/i915_gem_region.h"
48
+#include "gt/intel_engine_user.h"
49
+#include "gt/intel_gt.h"
50
+#include "gt/intel_gt_pm.h"
51
+#include "gt/intel_workarounds.h"
56
52
 
57
-	if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE))
58
-		return true;
53
+#include "i915_drv.h"
54
+#include "i915_trace.h"
55
+#include "i915_vgpu.h"
59
56
 
60
-	return obj->pin_global; /* currently in use by HW, keep flushed */
61
-}
57
+#include "intel_pm.h"
62
58
 
63
59
 static int
64
-insert_mappable_node(struct i915_ggtt *ggtt,
65
-                     struct drm_mm_node *node, u32 size)
60
+insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size)
66
61
 {
62
+	int err;
63
+
64
+	err = mutex_lock_interruptible(&ggtt->vm.mutex);
65
+	if (err)
66
+		return err;
67
+
67
68
 	memset(node, 0, sizeof(*node));
68
-	return drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
69
-					   size, 0, I915_COLOR_UNEVICTABLE,
70
-					   0, ggtt->mappable_end,
71
-					   DRM_MM_INSERT_LOW);
69
+	err = drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
70
+					  size, 0, I915_COLOR_UNEVICTABLE,
71
+					  0, ggtt->mappable_end,
72
+					  DRM_MM_INSERT_LOW);
73
+
74
+	mutex_unlock(&ggtt->vm.mutex);
75
+
76
+	return err;
72
77
 }
73
78
 
74
79
 static void
75
-remove_mappable_node(struct drm_mm_node *node)
80
+remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node)
76
81
 {
82
+	mutex_lock(&ggtt->vm.mutex);
77
83
 	drm_mm_remove_node(node);
78
-}
79
-
80
-/* some bookkeeping */
81
-static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
82
-				  u64 size)
83
-{
84
-	spin_lock(&dev_priv->mm.object_stat_lock);
85
-	dev_priv->mm.object_count++;
86
-	dev_priv->mm.object_memory += size;
87
-	spin_unlock(&dev_priv->mm.object_stat_lock);
88
-}
89
-
90
-static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
91
-				     u64 size)
92
-{
93
-	spin_lock(&dev_priv->mm.object_stat_lock);
94
-	dev_priv->mm.object_count--;
95
-	dev_priv->mm.object_memory -= size;
96
-	spin_unlock(&dev_priv->mm.object_stat_lock);
97
-}
98
-
99
-static int
100
-i915_gem_wait_for_error(struct i915_gpu_error *error)
101
-{
102
-	int ret;
103
-
104
-	might_sleep();
105
-
106
-	/*
107
-	 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
108
-	 * userspace. If it takes that long something really bad is going on and
109
-	 * we should simply try to bail out and fail as gracefully as possible.
110
-	 */
111
-	ret = wait_event_interruptible_timeout(error->reset_queue,
112
-					       !i915_reset_backoff(error),
113
-					       I915_RESET_TIMEOUT);
114
-	if (ret == 0) {
115
-		DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
116
-		return -EIO;
117
-	} else if (ret < 0) {
118
-		return ret;
119
-	} else {
120
-		return 0;
121
-	}
122
-}
123
-
124
-int i915_mutex_lock_interruptible(struct drm_device *dev)
125
-{
126
-	struct drm_i915_private *dev_priv = to_i915(dev);
127
-	int ret;
128
-
129
-	ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
130
-	if (ret)
131
-		return ret;
132
-
133
-	ret = mutex_lock_interruptible(&dev->struct_mutex);
134
-	if (ret)
135
-		return ret;
136
-
137
-	return 0;
138
-}
139
-
140
-static u32 __i915_gem_park(struct drm_i915_private *i915)
141
-{
142
-	GEM_TRACE("\n");
143
-
144
-	lockdep_assert_held(&i915->drm.struct_mutex);
145
-	GEM_BUG_ON(i915->gt.active_requests);
146
-	GEM_BUG_ON(!list_empty(&i915->gt.active_rings));
147
-
148
-	if (!i915->gt.awake)
149
-		return I915_EPOCH_INVALID;
150
-
151
-	GEM_BUG_ON(i915->gt.epoch == I915_EPOCH_INVALID);
152
-
153
-	/*
154
-	 * Be paranoid and flush a concurrent interrupt to make sure
155
-	 * we don't reactivate any irq tasklets after parking.
156
-	 *
157
-	 * FIXME: Note that even though we have waited for execlists to be idle,
158
-	 * there may still be an in-flight interrupt even though the CSB
159
-	 * is now empty. synchronize_irq() makes sure that a residual interrupt
160
-	 * is completed before we continue, but it doesn't prevent the HW from
161
-	 * raising a spurious interrupt later. To complete the shield we should
162
-	 * coordinate disabling the CS irq with flushing the interrupts.
163
-	 */
164
-	synchronize_irq(i915->drm.irq);
165
-
166
-	intel_engines_park(i915);
167
-	i915_timelines_park(i915);
168
-
169
-	i915_pmu_gt_parked(i915);
170
-	i915_vma_parked(i915);
171
-
172
-	i915->gt.awake = false;
173
-
174
-	if (INTEL_GEN(i915) >= 6)
175
-		gen6_rps_idle(i915);
176
-
177
-	if (NEEDS_RC6_CTX_CORRUPTION_WA(i915)) {
178
-		i915_rc6_ctx_wa_check(i915);
179
-		intel_uncore_forcewake_put(i915, FORCEWAKE_ALL);
180
-	}
181
-
182
-	intel_display_power_put(i915, POWER_DOMAIN_GT_IRQ);
183
-
184
-	intel_runtime_pm_put(i915);
185
-
186
-	return i915->gt.epoch;
187
-}
188
-
189
-void i915_gem_park(struct drm_i915_private *i915)
190
-{
191
-	GEM_TRACE("\n");
192
-
193
-	lockdep_assert_held(&i915->drm.struct_mutex);
194
-	GEM_BUG_ON(i915->gt.active_requests);
195
-
196
-	if (!i915->gt.awake)
197
-		return;
198
-
199
-	/* Defer the actual call to __i915_gem_park() to prevent ping-pongs */
200
-	mod_delayed_work(i915->wq, &i915->gt.idle_work, msecs_to_jiffies(100));
201
-}
202
-
203
-void i915_gem_unpark(struct drm_i915_private *i915)
204
-{
205
-	GEM_TRACE("\n");
206
-
207
-	lockdep_assert_held(&i915->drm.struct_mutex);
208
-	GEM_BUG_ON(!i915->gt.active_requests);
209
-
210
-	if (i915->gt.awake)
211
-		return;
212
-
213
-	intel_runtime_pm_get_noresume(i915);
214
-
215
-	/*
216
-	 * It seems that the DMC likes to transition between the DC states a lot
217
-	 * when there are no connected displays (no active power domains) during
218
-	 * command submission.
219
-	 *
220
-	 * This activity has negative impact on the performance of the chip with
221
-	 * huge latencies observed in the interrupt handler and elsewhere.
222
-	 *
223
-	 * Work around it by grabbing a GT IRQ power domain whilst there is any
224
-	 * GT activity, preventing any DC state transitions.
225
-	 */
226
-	intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
227
-
228
-	if (NEEDS_RC6_CTX_CORRUPTION_WA(i915))
229
-		intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
230
-
231
-	i915->gt.awake = true;
232
-	if (unlikely(++i915->gt.epoch == 0)) /* keep 0 as invalid */
233
-		i915->gt.epoch = 1;
234
-
235
-	intel_enable_gt_powersave(i915);
236
-	i915_update_gfx_val(i915);
237
-	if (INTEL_GEN(i915) >= 6)
238
-		gen6_rps_busy(i915);
239
-	i915_pmu_gt_unparked(i915);
240
-
241
-	intel_engines_unpark(i915);
242
-
243
-	i915_queue_hangcheck(i915);
244
-
245
-	queue_delayed_work(i915->wq,
246
-			   &i915->gt.retire_work,
247
-			   round_jiffies_up_relative(HZ));
84
+	mutex_unlock(&ggtt->vm.mutex);
248
85
 }
249
86
 
250
87
 int
251
88
 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
252
89
 			    struct drm_file *file)
253
90
 {
254
-	struct drm_i915_private *dev_priv = to_i915(dev);
255
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
91
+	struct i915_ggtt *ggtt = &to_i915(dev)->ggtt;
256
92
 	struct drm_i915_gem_get_aperture *args = data;
257
93
 	struct i915_vma *vma;
258
94
 	u64 pinned;
259
95
 
96
+	if (mutex_lock_interruptible(&ggtt->vm.mutex))
97
+		return -EINTR;
98
+
260
99
 	pinned = ggtt->vm.reserved;
261
-	mutex_lock(&dev->struct_mutex);
262
-	list_for_each_entry(vma, &ggtt->vm.active_list, vm_link)
100
+	list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
263
101
 		if (i915_vma_is_pinned(vma))
264
102
 			pinned += vma->node.size;
265
-	list_for_each_entry(vma, &ggtt->vm.inactive_list, vm_link)
266
-		if (i915_vma_is_pinned(vma))
267
-			pinned += vma->node.size;
268
-	mutex_unlock(&dev->struct_mutex);
103
+
104
+	mutex_unlock(&ggtt->vm.mutex);
269
105
 
270
106
 	args->aper_size = ggtt->vm.total;
271
107
 	args->aper_available_size = args->aper_size - pinned;
..
..
@@ -273,465 +109,97 @@
273
109
 	return 0;
274
110
 }
275
111
 
276
-static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
112
+int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
113
+			   unsigned long flags)
277
114
 {
278
-	struct address_space *mapping = obj->base.filp->f_mapping;
279
-	drm_dma_handle_t *phys;
280
-	struct sg_table *st;
281
-	struct scatterlist *sg;
282
-	char *vaddr;
283
-	int i;
284
-	int err;
285
-
286
-	if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
287
-		return -EINVAL;
288
-
289
-	/* Always aligning to the object size, allows a single allocation
290
-	 * to handle all possible callers, and given typical object sizes,
291
-	 * the alignment of the buddy allocation will naturally match.
292
-	 */
293
-	phys = drm_pci_alloc(obj->base.dev,
294
-			     roundup_pow_of_two(obj->base.size),
295
-			     roundup_pow_of_two(obj->base.size));
296
-	if (!phys)
297
-		return -ENOMEM;
298
-
299
-	vaddr = phys->vaddr;
300
-	for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
301
-		struct page *page;
302
-		char *src;
303
-
304
-		page = shmem_read_mapping_page(mapping, i);
305
-		if (IS_ERR(page)) {
306
-			err = PTR_ERR(page);
307
-			goto err_phys;
308
-		}
309
-
310
-		src = kmap_atomic(page);
311
-		memcpy(vaddr, src, PAGE_SIZE);
312
-		drm_clflush_virt_range(vaddr, PAGE_SIZE);
313
-		kunmap_atomic(src);
314
-
315
-		put_page(page);
316
-		vaddr += PAGE_SIZE;
317
-	}
318
-
319
-	i915_gem_chipset_flush(to_i915(obj->base.dev));
320
-
321
-	st = kmalloc(sizeof(*st), GFP_KERNEL);
322
-	if (!st) {
323
-		err = -ENOMEM;
324
-		goto err_phys;
325
-	}
326
-
327
-	if (sg_alloc_table(st, 1, GFP_KERNEL)) {
328
-		kfree(st);
329
-		err = -ENOMEM;
330
-		goto err_phys;
331
-	}
332
-
333
-	sg = st->sgl;
334
-	sg->offset = 0;
335
-	sg->length = obj->base.size;
336
-
337
-	sg_dma_address(sg) = phys->busaddr;
338
-	sg_dma_len(sg) = obj->base.size;
339
-
340
-	obj->phys_handle = phys;
341
-
342
-	__i915_gem_object_set_pages(obj, st, sg->length);
343
-
344
-	return 0;
345
-
346
-err_phys:
347
-	drm_pci_free(obj->base.dev, phys);
348
-
349
-	return err;
350
-}
351
-
352
-static void __start_cpu_write(struct drm_i915_gem_object *obj)
353
-{
354
-	obj->read_domains = I915_GEM_DOMAIN_CPU;
355
-	obj->write_domain = I915_GEM_DOMAIN_CPU;
356
-	if (cpu_write_needs_clflush(obj))
357
-		obj->cache_dirty = true;
358
-}
359
-
360
-static void
361
-__i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
362
-				struct sg_table *pages,
363
-				bool needs_clflush)
364
-{
365
-	GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
366
-
367
-	if (obj->mm.madv == I915_MADV_DONTNEED)
368
-		obj->mm.dirty = false;
369
-
370
-	if (needs_clflush &&
371
-	    (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0 &&
372
-	    !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
373
-		drm_clflush_sg(pages);
374
-
375
-	__start_cpu_write(obj);
376
-}
377
-
378
-static void
379
-i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
380
-			       struct sg_table *pages)
381
-{
382
-	__i915_gem_object_release_shmem(obj, pages, false);
383
-
384
-	if (obj->mm.dirty) {
385
-		struct address_space *mapping = obj->base.filp->f_mapping;
386
-		char *vaddr = obj->phys_handle->vaddr;
387
-		int i;
388
-
389
-		for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
390
-			struct page *page;
391
-			char *dst;
392
-
393
-			page = shmem_read_mapping_page(mapping, i);
394
-			if (IS_ERR(page))
395
-				continue;
396
-
397
-			dst = kmap_atomic(page);
398
-			drm_clflush_virt_range(vaddr, PAGE_SIZE);
399
-			memcpy(dst, vaddr, PAGE_SIZE);
400
-			kunmap_atomic(dst);
401
-
402
-			set_page_dirty(page);
403
-			if (obj->mm.madv == I915_MADV_WILLNEED)
404
-				mark_page_accessed(page);
405
-			put_page(page);
406
-			vaddr += PAGE_SIZE;
407
-		}
408
-		obj->mm.dirty = false;
409
-	}
410
-
411
-	sg_free_table(pages);
412
-	kfree(pages);
413
-
414
-	drm_pci_free(obj->base.dev, obj->phys_handle);
415
-}
416
-
417
-static void
418
-i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
419
-{
420
-	i915_gem_object_unpin_pages(obj);
421
-}
422
-
423
-static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
424
-	.get_pages = i915_gem_object_get_pages_phys,
425
-	.put_pages = i915_gem_object_put_pages_phys,
426
-	.release = i915_gem_object_release_phys,
427
-};
428
-
429
-static const struct drm_i915_gem_object_ops i915_gem_object_ops;
430
-
431
-int i915_gem_object_unbind(struct drm_i915_gem_object *obj)
432
-{
433
-	struct i915_vma *vma;
115
+	struct intel_runtime_pm *rpm = &to_i915(obj->base.dev)->runtime_pm;
434
116
 	LIST_HEAD(still_in_list);
117
+	intel_wakeref_t wakeref;
118
+	struct i915_vma *vma;
435
119
 	int ret;
436
120
 
437
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
121
+	if (list_empty(&obj->vma.list))
122
+		return 0;
438
123
 
439
-	/* Closed vma are removed from the obj->vma_list - but they may
440
-	 * still have an active binding on the object. To remove those we
441
-	 * must wait for all rendering to complete to the object (as unbinding
442
-	 * must anyway), and retire the requests.
124
+	/*
125
+	 * As some machines use ACPI to handle runtime-resume callbacks, and
126
+	 * ACPI is quite kmalloc happy, we cannot resume beneath the vm->mutex
127
+	 * as they are required by the shrinker. Ergo, we wake the device up
128
+	 * first just in case.
443
129
 	 */
444
-	ret = i915_gem_object_set_to_cpu_domain(obj, false);
445
-	if (ret)
446
-		return ret;
130
+	wakeref = intel_runtime_pm_get(rpm);
447
131
 
448
-	while ((vma = list_first_entry_or_null(&obj->vma_list,
449
-					       struct i915_vma,
450
-					       obj_link))) {
132
+try_again:
133
+	ret = 0;
134
+	spin_lock(&obj->vma.lock);
135
+	while (!ret && (vma = list_first_entry_or_null(&obj->vma.list,
136
+						       struct i915_vma,
137
+						       obj_link))) {
138
+		struct i915_address_space *vm = vma->vm;
139
+
451
140
 		list_move_tail(&vma->obj_link, &still_in_list);
452
-		ret = i915_vma_unbind(vma);
453
-		if (ret)
141
+		if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK))
142
+			continue;
143
+
144
+		if (flags & I915_GEM_OBJECT_UNBIND_TEST) {
145
+			ret = -EBUSY;
454
146
 			break;
147
+		}
148
+
149
+		ret = -EAGAIN;
150
+		if (!i915_vm_tryopen(vm))
151
+			break;
152
+
153
+		/* Prevent vma being freed by i915_vma_parked as we unbind */
154
+		vma = __i915_vma_get(vma);
155
+		spin_unlock(&obj->vma.lock);
156
+
157
+		if (vma) {
158
+			ret = -EBUSY;
159
+			if (flags & I915_GEM_OBJECT_UNBIND_ACTIVE ||
160
+			    !i915_vma_is_active(vma))
161
+				ret = i915_vma_unbind(vma);
162
+
163
+			__i915_vma_put(vma);
164
+		}
165
+
166
+		i915_vm_close(vm);
167
+		spin_lock(&obj->vma.lock);
455
168
 	}
456
-	list_splice(&still_in_list, &obj->vma_list);
169
+	list_splice_init(&still_in_list, &obj->vma.list);
170
+	spin_unlock(&obj->vma.lock);
171
+
172
+	if (ret == -EAGAIN && flags & I915_GEM_OBJECT_UNBIND_BARRIER) {
173
+		rcu_barrier(); /* flush the i915_vm_release() */
174
+		goto try_again;
175
+	}
176
+
177
+	intel_runtime_pm_put(rpm, wakeref);
457
178
 
458
179
 	return ret;
459
180
 }
460
181
 
461
-static long
462
-i915_gem_object_wait_fence(struct dma_fence *fence,
463
-			   unsigned int flags,
464
-			   long timeout,
465
-			   struct intel_rps_client *rps_client)
466
-{
467
-	struct i915_request *rq;
468
-
469
-	BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
470
-
471
-	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
472
-		return timeout;
473
-
474
-	if (!dma_fence_is_i915(fence))
475
-		return dma_fence_wait_timeout(fence,
476
-					      flags & I915_WAIT_INTERRUPTIBLE,
477
-					      timeout);
478
-
479
-	rq = to_request(fence);
480
-	if (i915_request_completed(rq))
481
-		goto out;
482
-
483
-	/*
484
-	 * This client is about to stall waiting for the GPU. In many cases
485
-	 * this is undesirable and limits the throughput of the system, as
486
-	 * many clients cannot continue processing user input/output whilst
487
-	 * blocked. RPS autotuning may take tens of milliseconds to respond
488
-	 * to the GPU load and thus incurs additional latency for the client.
489
-	 * We can circumvent that by promoting the GPU frequency to maximum
490
-	 * before we wait. This makes the GPU throttle up much more quickly
491
-	 * (good for benchmarks and user experience, e.g. window animations),
492
-	 * but at a cost of spending more power processing the workload
493
-	 * (bad for battery). Not all clients even want their results
494
-	 * immediately and for them we should just let the GPU select its own
495
-	 * frequency to maximise efficiency. To prevent a single client from
496
-	 * forcing the clocks too high for the whole system, we only allow
497
-	 * each client to waitboost once in a busy period.
498
-	 */
499
-	if (rps_client && !i915_request_started(rq)) {
500
-		if (INTEL_GEN(rq->i915) >= 6)
501
-			gen6_rps_boost(rq, rps_client);
502
-	}
503
-
504
-	timeout = i915_request_wait(rq, flags, timeout);
505
-
506
-out:
507
-	if (flags & I915_WAIT_LOCKED && i915_request_completed(rq))
508
-		i915_request_retire_upto(rq);
509
-
510
-	return timeout;
511
-}
512
-
513
-static long
514
-i915_gem_object_wait_reservation(struct reservation_object *resv,
515
-				 unsigned int flags,
516
-				 long timeout,
517
-				 struct intel_rps_client *rps_client)
518
-{
519
-	unsigned int seq = read_seqbegin(&resv->seq);
520
-	struct dma_fence *excl;
521
-	bool prune_fences = false;
522
-
523
-	if (flags & I915_WAIT_ALL) {
524
-		struct dma_fence **shared;
525
-		unsigned int count, i;
526
-		int ret;
527
-
528
-		ret = reservation_object_get_fences_rcu(resv,
529
-							&excl, &count, &shared);
530
-		if (ret)
531
-			return ret;
532
-
533
-		for (i = 0; i < count; i++) {
534
-			timeout = i915_gem_object_wait_fence(shared[i],
535
-							     flags, timeout,
536
-							     rps_client);
537
-			if (timeout < 0)
538
-				break;
539
-
540
-			dma_fence_put(shared[i]);
541
-		}
542
-
543
-		for (; i < count; i++)
544
-			dma_fence_put(shared[i]);
545
-		kfree(shared);
546
-
547
-		/*
548
-		 * If both shared fences and an exclusive fence exist,
549
-		 * then by construction the shared fences must be later
550
-		 * than the exclusive fence. If we successfully wait for
551
-		 * all the shared fences, we know that the exclusive fence
552
-		 * must all be signaled. If all the shared fences are
553
-		 * signaled, we can prune the array and recover the
554
-		 * floating references on the fences/requests.
555
-		 */
556
-		prune_fences = count && timeout >= 0;
557
-	} else {
558
-		excl = reservation_object_get_excl_rcu(resv);
559
-	}
560
-
561
-	if (excl && timeout >= 0)
562
-		timeout = i915_gem_object_wait_fence(excl, flags, timeout,
563
-						     rps_client);
564
-
565
-	dma_fence_put(excl);
566
-
567
-	/*
568
-	 * Opportunistically prune the fences iff we know they have *all* been
569
-	 * signaled and that the reservation object has not been changed (i.e.
570
-	 * no new fences have been added).
571
-	 */
572
-	if (prune_fences && !read_seqretry(&resv->seq, seq)) {
573
-		if (reservation_object_trylock(resv)) {
574
-			if (!read_seqretry(&resv->seq, seq))
575
-				reservation_object_add_excl_fence(resv, NULL);
576
-			reservation_object_unlock(resv);
577
-		}
578
-	}
579
-
580
-	return timeout;
581
-}
582
-
583
-static void __fence_set_priority(struct dma_fence *fence,
584
-				 const struct i915_sched_attr *attr)
585
-{
586
-	struct i915_request *rq;
587
-	struct intel_engine_cs *engine;
588
-
589
-	if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence))
590
-		return;
591
-
592
-	rq = to_request(fence);
593
-	engine = rq->engine;
594
-
595
-	local_bh_disable();
596
-	rcu_read_lock(); /* RCU serialisation for set-wedged protection */
597
-	if (engine->schedule)
598
-		engine->schedule(rq, attr);
599
-	rcu_read_unlock();
600
-	local_bh_enable(); /* kick the tasklets if queues were reprioritised */
601
-}
602
-
603
-static void fence_set_priority(struct dma_fence *fence,
604
-			       const struct i915_sched_attr *attr)
605
-{
606
-	/* Recurse once into a fence-array */
607
-	if (dma_fence_is_array(fence)) {
608
-		struct dma_fence_array *array = to_dma_fence_array(fence);
609
-		int i;
610
-
611
-		for (i = 0; i < array->num_fences; i++)
612
-			__fence_set_priority(array->fences[i], attr);
613
-	} else {
614
-		__fence_set_priority(fence, attr);
615
-	}
616
-}
617
-
618
-int
619
-i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
620
-			      unsigned int flags,
621
-			      const struct i915_sched_attr *attr)
622
-{
623
-	struct dma_fence *excl;
624
-
625
-	if (flags & I915_WAIT_ALL) {
626
-		struct dma_fence **shared;
627
-		unsigned int count, i;
628
-		int ret;
629
-
630
-		ret = reservation_object_get_fences_rcu(obj->resv,
631
-							&excl, &count, &shared);
632
-		if (ret)
633
-			return ret;
634
-
635
-		for (i = 0; i < count; i++) {
636
-			fence_set_priority(shared[i], attr);
637
-			dma_fence_put(shared[i]);
638
-		}
639
-
640
-		kfree(shared);
641
-	} else {
642
-		excl = reservation_object_get_excl_rcu(obj->resv);
643
-	}
644
-
645
-	if (excl) {
646
-		fence_set_priority(excl, attr);
647
-		dma_fence_put(excl);
648
-	}
649
-	return 0;
650
-}
651
-
652
-/**
653
- * Waits for rendering to the object to be completed
654
- * @obj: i915 gem object
655
- * @flags: how to wait (under a lock, for all rendering or just for writes etc)
656
- * @timeout: how long to wait
657
- * @rps_client: client (user process) to charge for any waitboosting
658
- */
659
-int
660
-i915_gem_object_wait(struct drm_i915_gem_object *obj,
661
-		     unsigned int flags,
662
-		     long timeout,
663
-		     struct intel_rps_client *rps_client)
664
-{
665
-	might_sleep();
666
-#if IS_ENABLED(CONFIG_LOCKDEP)
667
-	GEM_BUG_ON(debug_locks &&
668
-		   !!lockdep_is_held(&obj->base.dev->struct_mutex) !=
669
-		   !!(flags & I915_WAIT_LOCKED));
670
-#endif
671
-	GEM_BUG_ON(timeout < 0);
672
-
673
-	timeout = i915_gem_object_wait_reservation(obj->resv,
674
-						   flags, timeout,
675
-						   rps_client);
676
-	return timeout < 0 ? timeout : 0;
677
-}
678
-
679
-static struct intel_rps_client *to_rps_client(struct drm_file *file)
680
-{
681
-	struct drm_i915_file_private *fpriv = file->driver_priv;
682
-
683
-	return &fpriv->rps_client;
684
-}
685
-
686
-static int
687
-i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
688
-		     struct drm_i915_gem_pwrite *args,
689
-		     struct drm_file *file)
690
-{
691
-	void *vaddr = obj->phys_handle->vaddr + args->offset;
692
-	char __user *user_data = u64_to_user_ptr(args->data_ptr);
693
-
694
-	/* We manually control the domain here and pretend that it
695
-	 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
696
-	 */
697
-	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
698
-	if (copy_from_user(vaddr, user_data, args->size))
699
-		return -EFAULT;
700
-
701
-	drm_clflush_virt_range(vaddr, args->size);
702
-	i915_gem_chipset_flush(to_i915(obj->base.dev));
703
-
704
-	intel_fb_obj_flush(obj, ORIGIN_CPU);
705
-	return 0;
706
-}
707
-
708
-void *i915_gem_object_alloc(struct drm_i915_private *dev_priv)
709
-{
710
-	return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
711
-}
712
-
713
-void i915_gem_object_free(struct drm_i915_gem_object *obj)
714
-{
715
-	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
716
-	kmem_cache_free(dev_priv->objects, obj);
717
-}
718
-
719
182
 static int
720
183
 i915_gem_create(struct drm_file *file,
721
-		struct drm_i915_private *dev_priv,
722
-		uint64_t size,
723
-		uint32_t *handle_p)
184
+		struct intel_memory_region *mr,
185
+		u64 *size_p,
186
+		u32 *handle_p)
724
187
 {
725
188
 	struct drm_i915_gem_object *obj;
726
-	int ret;
727
189
 	u32 handle;
190
+	u64 size;
191
+	int ret;
728
192
 
729
-	size = roundup(size, PAGE_SIZE);
193
+	GEM_BUG_ON(!is_power_of_2(mr->min_page_size));
194
+	size = round_up(*size_p, mr->min_page_size);
730
195
 	if (size == 0)
731
196
 		return -EINVAL;
732
197
 
198
+	/* For most of the ABI (e.g. mmap) we think in system pages */
199
+	GEM_BUG_ON(!IS_ALIGNED(size, PAGE_SIZE));
200
+
733
201
 	/* Allocate the new object */
734
-	obj = i915_gem_object_create(dev_priv, size);
202
+	obj = i915_gem_object_create_region(mr, size, 0);
735
203
 	if (IS_ERR(obj))
736
204
 		return PTR_ERR(obj);
737
205
 
..
..
@@ -742,6 +210,7 @@
742
210
 		return ret;
743
211
 
744
212
 	*handle_p = handle;
213
+	*size_p = size;
745
214
 	return 0;
746
215
 }
747
216
 
..
..
@@ -750,17 +219,45 @@
750
219
 		     struct drm_device *dev,
751
220
 		     struct drm_mode_create_dumb *args)
752
221
 {
753
-	/* have to work out size/pitch and return them */
754
-	args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
755
-	args->size = args->pitch * args->height;
756
-	return i915_gem_create(file, to_i915(dev),
757
-			       args->size, &args->handle);
758
-}
222
+	enum intel_memory_type mem_type;
223
+	int cpp = DIV_ROUND_UP(args->bpp, 8);
224
+	u32 format;
759
225
 
760
-static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj)
761
-{
762
-	return !(obj->cache_level == I915_CACHE_NONE ||
763
-		 obj->cache_level == I915_CACHE_WT);
226
+	switch (cpp) {
227
+	case 1:
228
+		format = DRM_FORMAT_C8;
229
+		break;
230
+	case 2:
231
+		format = DRM_FORMAT_RGB565;
232
+		break;
233
+	case 4:
234
+		format = DRM_FORMAT_XRGB8888;
235
+		break;
236
+	default:
237
+		return -EINVAL;
238
+	}
239
+
240
+	/* have to work out size/pitch and return them */
241
+	args->pitch = ALIGN(args->width * cpp, 64);
242
+
243
+	/* align stride to page size so that we can remap */
244
+	if (args->pitch > intel_plane_fb_max_stride(to_i915(dev), format,
245
+						    DRM_FORMAT_MOD_LINEAR))
246
+		args->pitch = ALIGN(args->pitch, 4096);
247
+
248
+	if (args->pitch < args->width)
249
+		return -EINVAL;
250
+
251
+	args->size = mul_u32_u32(args->pitch, args->height);
252
+
253
+	mem_type = INTEL_MEMORY_SYSTEM;
254
+	if (HAS_LMEM(to_i915(dev)))
255
+		mem_type = INTEL_MEMORY_LOCAL;
256
+
257
+	return i915_gem_create(file,
258
+			       intel_memory_region_by_type(to_i915(dev),
259
+							   mem_type),
260
+			       &args->size, &args->handle);
764
261
 }
765
262
 
766
263
 /**
..
..
@@ -773,357 +270,63 @@
773
270
 i915_gem_create_ioctl(struct drm_device *dev, void *data,
774
271
 		      struct drm_file *file)
775
272
 {
776
-	struct drm_i915_private *dev_priv = to_i915(dev);
273
+	struct drm_i915_private *i915 = to_i915(dev);
777
274
 	struct drm_i915_gem_create *args = data;
778
275
 
779
-	i915_gem_flush_free_objects(dev_priv);
276
+	i915_gem_flush_free_objects(i915);
780
277
 
781
-	return i915_gem_create(file, dev_priv,
782
-			       args->size, &args->handle);
278
+	return i915_gem_create(file,
279
+			       intel_memory_region_by_type(i915,
280
+							   INTEL_MEMORY_SYSTEM),
281
+			       &args->size, &args->handle);
783
282
 }
784
283
 
785
-static inline enum fb_op_origin
786
-fb_write_origin(struct drm_i915_gem_object *obj, unsigned int domain)
787
-{
788
-	return (domain == I915_GEM_DOMAIN_GTT ?
789
-		obj->frontbuffer_ggtt_origin : ORIGIN_CPU);
790
-}
791
-
792
-void i915_gem_flush_ggtt_writes(struct drm_i915_private *dev_priv)
793
-{
794
-	/*
795
-	 * No actual flushing is required for the GTT write domain for reads
796
-	 * from the GTT domain. Writes to it "immediately" go to main memory
797
-	 * as far as we know, so there's no chipset flush. It also doesn't
798
-	 * land in the GPU render cache.
799
-	 *
800
-	 * However, we do have to enforce the order so that all writes through
801
-	 * the GTT land before any writes to the device, such as updates to
802
-	 * the GATT itself.
803
-	 *
804
-	 * We also have to wait a bit for the writes to land from the GTT.
805
-	 * An uncached read (i.e. mmio) seems to be ideal for the round-trip
806
-	 * timing. This issue has only been observed when switching quickly
807
-	 * between GTT writes and CPU reads from inside the kernel on recent hw,
808
-	 * and it appears to only affect discrete GTT blocks (i.e. on LLC
809
-	 * system agents we cannot reproduce this behaviour, until Cannonlake
810
-	 * that was!).
811
-	 */
812
-
813
-	i915_gem_chipset_flush(dev_priv);
814
-
815
-	intel_runtime_pm_get(dev_priv);
816
-	spin_lock_irq(&dev_priv->uncore.lock);
817
-
818
-	POSTING_READ_FW(RING_HEAD(RENDER_RING_BASE));
819
-
820
-	spin_unlock_irq(&dev_priv->uncore.lock);
821
-	intel_runtime_pm_put(dev_priv);
822
-}
823
-
824
-static void
825
-flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)
826
-{
827
-	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
828
-	struct i915_vma *vma;
829
-
830
-	if (!(obj->write_domain & flush_domains))
831
-		return;
832
-
833
-	switch (obj->write_domain) {
834
-	case I915_GEM_DOMAIN_GTT:
835
-		i915_gem_flush_ggtt_writes(dev_priv);
836
-
837
-		intel_fb_obj_flush(obj,
838
-				   fb_write_origin(obj, I915_GEM_DOMAIN_GTT));
839
-
840
-		for_each_ggtt_vma(vma, obj) {
841
-			if (vma->iomap)
842
-				continue;
843
-
844
-			i915_vma_unset_ggtt_write(vma);
845
-		}
846
-		break;
847
-
848
-	case I915_GEM_DOMAIN_WC:
849
-		wmb();
850
-		break;
851
-
852
-	case I915_GEM_DOMAIN_CPU:
853
-		i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
854
-		break;
855
-
856
-	case I915_GEM_DOMAIN_RENDER:
857
-		if (gpu_write_needs_clflush(obj))
858
-			obj->cache_dirty = true;
859
-		break;
860
-	}
861
-
862
-	obj->write_domain = 0;
863
-}
864
-
865
-static inline int
866
-__copy_to_user_swizzled(char __user *cpu_vaddr,
867
-			const char *gpu_vaddr, int gpu_offset,
868
-			int length)
869
-{
870
-	int ret, cpu_offset = 0;
871
-
872
-	while (length > 0) {
873
-		int cacheline_end = ALIGN(gpu_offset + 1, 64);
874
-		int this_length = min(cacheline_end - gpu_offset, length);
875
-		int swizzled_gpu_offset = gpu_offset ^ 64;
876
-
877
-		ret = __copy_to_user(cpu_vaddr + cpu_offset,
878
-				     gpu_vaddr + swizzled_gpu_offset,
879
-				     this_length);
880
-		if (ret)
881
-			return ret + length;
882
-
883
-		cpu_offset += this_length;
884
-		gpu_offset += this_length;
885
-		length -= this_length;
886
-	}
887
-
888
-	return 0;
889
-}
890
-
891
-static inline int
892
-__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
893
-			  const char __user *cpu_vaddr,
894
-			  int length)
895
-{
896
-	int ret, cpu_offset = 0;
897
-
898
-	while (length > 0) {
899
-		int cacheline_end = ALIGN(gpu_offset + 1, 64);
900
-		int this_length = min(cacheline_end - gpu_offset, length);
901
-		int swizzled_gpu_offset = gpu_offset ^ 64;
902
-
903
-		ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
904
-				       cpu_vaddr + cpu_offset,
905
-				       this_length);
906
-		if (ret)
907
-			return ret + length;
908
-
909
-		cpu_offset += this_length;
910
-		gpu_offset += this_length;
911
-		length -= this_length;
912
-	}
913
-
914
-	return 0;
915
-}
916
-
917
-/*
918
- * Pins the specified object's pages and synchronizes the object with
919
- * GPU accesses. Sets needs_clflush to non-zero if the caller should
920
- * flush the object from the CPU cache.
921
- */
922
-int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
923
-				    unsigned int *needs_clflush)
924
-{
925
-	int ret;
926
-
927
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
928
-
929
-	*needs_clflush = 0;
930
-	if (!i915_gem_object_has_struct_page(obj))
931
-		return -ENODEV;
932
-
933
-	ret = i915_gem_object_wait(obj,
934
-				   I915_WAIT_INTERRUPTIBLE |
935
-				   I915_WAIT_LOCKED,
936
-				   MAX_SCHEDULE_TIMEOUT,
937
-				   NULL);
938
-	if (ret)
939
-		return ret;
940
-
941
-	ret = i915_gem_object_pin_pages(obj);
942
-	if (ret)
943
-		return ret;
944
-
945
-	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||
946
-	    !static_cpu_has(X86_FEATURE_CLFLUSH)) {
947
-		ret = i915_gem_object_set_to_cpu_domain(obj, false);
948
-		if (ret)
949
-			goto err_unpin;
950
-		else
951
-			goto out;
952
-	}
953
-
954
-	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
955
-
956
-	/* If we're not in the cpu read domain, set ourself into the gtt
957
-	 * read domain and manually flush cachelines (if required). This
958
-	 * optimizes for the case when the gpu will dirty the data
959
-	 * anyway again before the next pread happens.
960
-	 */
961
-	if (!obj->cache_dirty &&
962
-	    !(obj->read_domains & I915_GEM_DOMAIN_CPU))
963
-		*needs_clflush = CLFLUSH_BEFORE;
964
-
965
-out:
966
-	/* return with the pages pinned */
967
-	return 0;
968
-
969
-err_unpin:
970
-	i915_gem_object_unpin_pages(obj);
971
-	return ret;
972
-}
973
-
974
-int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
975
-				     unsigned int *needs_clflush)
976
-{
977
-	int ret;
978
-
979
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
980
-
981
-	*needs_clflush = 0;
982
-	if (!i915_gem_object_has_struct_page(obj))
983
-		return -ENODEV;
984
-
985
-	ret = i915_gem_object_wait(obj,
986
-				   I915_WAIT_INTERRUPTIBLE |
987
-				   I915_WAIT_LOCKED |
988
-				   I915_WAIT_ALL,
989
-				   MAX_SCHEDULE_TIMEOUT,
990
-				   NULL);
991
-	if (ret)
992
-		return ret;
993
-
994
-	ret = i915_gem_object_pin_pages(obj);
995
-	if (ret)
996
-		return ret;
997
-
998
-	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||
999
-	    !static_cpu_has(X86_FEATURE_CLFLUSH)) {
1000
-		ret = i915_gem_object_set_to_cpu_domain(obj, true);
1001
-		if (ret)
1002
-			goto err_unpin;
1003
-		else
1004
-			goto out;
1005
-	}
1006
-
1007
-	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
1008
-
1009
-	/* If we're not in the cpu write domain, set ourself into the
1010
-	 * gtt write domain and manually flush cachelines (as required).
1011
-	 * This optimizes for the case when the gpu will use the data
1012
-	 * right away and we therefore have to clflush anyway.
1013
-	 */
1014
-	if (!obj->cache_dirty) {
1015
-		*needs_clflush |= CLFLUSH_AFTER;
1016
-
1017
-		/*
1018
-		 * Same trick applies to invalidate partially written
1019
-		 * cachelines read before writing.
1020
-		 */
1021
-		if (!(obj->read_domains & I915_GEM_DOMAIN_CPU))
1022
-			*needs_clflush |= CLFLUSH_BEFORE;
1023
-	}
1024
-
1025
-out:
1026
-	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
1027
-	obj->mm.dirty = true;
1028
-	/* return with the pages pinned */
1029
-	return 0;
1030
-
1031
-err_unpin:
1032
-	i915_gem_object_unpin_pages(obj);
1033
-	return ret;
1034
-}
1035
-
1036
-static void
1037
-shmem_clflush_swizzled_range(char *addr, unsigned long length,
1038
-			     bool swizzled)
1039
-{
1040
-	if (unlikely(swizzled)) {
1041
-		unsigned long start = (unsigned long) addr;
1042
-		unsigned long end = (unsigned long) addr + length;
1043
-
1044
-		/* For swizzling simply ensure that we always flush both
1045
-		 * channels. Lame, but simple and it works. Swizzled
1046
-		 * pwrite/pread is far from a hotpath - current userspace
1047
-		 * doesn't use it at all. */
1048
-		start = round_down(start, 128);
1049
-		end = round_up(end, 128);
1050
-
1051
-		drm_clflush_virt_range((void *)start, end - start);
1052
-	} else {
1053
-		drm_clflush_virt_range(addr, length);
1054
-	}
1055
-
1056
-}
1057
-
1058
-/* Only difference to the fast-path function is that this can handle bit17
1059
- * and uses non-atomic copy and kmap functions. */
1060
284
 static int
1061
-shmem_pread_slow(struct page *page, int offset, int length,
1062
-		 char __user *user_data,
1063
-		 bool page_do_bit17_swizzling, bool needs_clflush)
285
+shmem_pread(struct page *page, int offset, int len, char __user *user_data,
286
+	    bool needs_clflush)
1064
287
 {
1065
288
 	char *vaddr;
1066
289
 	int ret;
1067
290
 
1068
291
 	vaddr = kmap(page);
1069
-	if (needs_clflush)
1070
-		shmem_clflush_swizzled_range(vaddr + offset, length,
1071
-					     page_do_bit17_swizzling);
1072
292
 
1073
-	if (page_do_bit17_swizzling)
1074
-		ret = __copy_to_user_swizzled(user_data, vaddr, offset, length);
1075
-	else
1076
-		ret = __copy_to_user(user_data, vaddr + offset, length);
293
+	if (needs_clflush)
294
+		drm_clflush_virt_range(vaddr + offset, len);
295
+
296
+	ret = __copy_to_user(user_data, vaddr + offset, len);
297
+
1077
298
 	kunmap(page);
1078
299
 
1079
-	return ret ? - EFAULT : 0;
1080
-}
1081
-
1082
-static int
1083
-shmem_pread(struct page *page, int offset, int length, char __user *user_data,
1084
-	    bool page_do_bit17_swizzling, bool needs_clflush)
1085
-{
1086
-	int ret;
1087
-
1088
-	ret = -ENODEV;
1089
-	if (!page_do_bit17_swizzling) {
1090
-		char *vaddr = kmap_atomic(page);
1091
-
1092
-		if (needs_clflush)
1093
-			drm_clflush_virt_range(vaddr + offset, length);
1094
-		ret = __copy_to_user_inatomic(user_data, vaddr + offset, length);
1095
-		kunmap_atomic(vaddr);
1096
-	}
1097
-	if (ret == 0)
1098
-		return 0;
1099
-
1100
-	return shmem_pread_slow(page, offset, length, user_data,
1101
-				page_do_bit17_swizzling, needs_clflush);
300
+	return ret ? -EFAULT : 0;
1102
301
 }
1103
302
 
1104
303
 static int
1105
304
 i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
1106
305
 		     struct drm_i915_gem_pread *args)
1107
306
 {
1108
-	char __user *user_data;
1109
-	u64 remain;
1110
-	unsigned int obj_do_bit17_swizzling;
1111
307
 	unsigned int needs_clflush;
1112
308
 	unsigned int idx, offset;
309
+	struct dma_fence *fence;
310
+	char __user *user_data;
311
+	u64 remain;
1113
312
 	int ret;
1114
313
 
1115
-	obj_do_bit17_swizzling = 0;
1116
-	if (i915_gem_object_needs_bit17_swizzle(obj))
1117
-		obj_do_bit17_swizzling = BIT(17);
1118
-
1119
-	ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex);
314
+	ret = i915_gem_object_lock_interruptible(obj, NULL);
1120
315
 	if (ret)
1121
316
 		return ret;
1122
317
 
1123
-	ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
1124
-	mutex_unlock(&obj->base.dev->struct_mutex);
1125
-	if (ret)
318
+	ret = i915_gem_object_prepare_read(obj, &needs_clflush);
319
+	if (ret) {
320
+		i915_gem_object_unlock(obj);
1126
321
 		return ret;
322
+	}
323
+
324
+	fence = i915_gem_object_lock_fence(obj);
325
+	i915_gem_object_finish_access(obj);
326
+	i915_gem_object_unlock(obj);
327
+
328
+	if (!fence)
329
+		return -ENOMEM;
1127
330
 
1128
331
 	remain = args->size;
1129
332
 	user_data = u64_to_user_ptr(args->data_ptr);
..
..
@@ -1133,7 +336,6 @@
1133
336
 		unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
1134
337
 
1135
338
 		ret = shmem_pread(page, offset, length, user_data,
1136
-				  page_to_phys(page) & obj_do_bit17_swizzling,
1137
339
 				  needs_clflush);
1138
340
 		if (ret)
1139
341
 			break;
..
..
@@ -1143,7 +345,7 @@
1143
345
 		offset = 0;
1144
346
 	}
1145
347
 
1146
-	i915_gem_obj_finish_shmem_access(obj);
348
+	i915_gem_object_unlock_fence(obj, fence);
1147
349
 	return ret;
1148
350
 }
1149
351
 
..
..
@@ -1177,42 +379,47 @@
1177
379
 {
1178
380
 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
1179
381
 	struct i915_ggtt *ggtt = &i915->ggtt;
382
+	intel_wakeref_t wakeref;
1180
383
 	struct drm_mm_node node;
1181
-	struct i915_vma *vma;
384
+	struct dma_fence *fence;
1182
385
 	void __user *user_data;
386
+	struct i915_vma *vma;
1183
387
 	u64 remain, offset;
1184
388
 	int ret;
1185
389
 
1186
-	ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
1187
-	if (ret)
1188
-		return ret;
1189
-
1190
-	intel_runtime_pm_get(i915);
1191
-	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
1192
-				       PIN_MAPPABLE |
1193
-				       PIN_NONFAULT |
1194
-				       PIN_NONBLOCK);
390
+	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
391
+	vma = ERR_PTR(-ENODEV);
392
+	if (!i915_gem_object_is_tiled(obj))
393
+		vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
394
+					       PIN_MAPPABLE |
395
+					       PIN_NONBLOCK /* NOWARN */ |
396
+					       PIN_NOEVICT);
1195
397
 	if (!IS_ERR(vma)) {
1196
398
 		node.start = i915_ggtt_offset(vma);
1197
-		node.allocated = false;
1198
-		ret = i915_vma_put_fence(vma);
1199
-		if (ret) {
1200
-			i915_vma_unpin(vma);
1201
-			vma = ERR_PTR(ret);
1202
-		}
1203
-	}
1204
-	if (IS_ERR(vma)) {
399
+		node.flags = 0;
400
+	} else {
1205
401
 		ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
1206
402
 		if (ret)
1207
-			goto out_unlock;
1208
-		GEM_BUG_ON(!node.allocated);
403
+			goto out_rpm;
404
+		GEM_BUG_ON(!drm_mm_node_allocated(&node));
1209
405
 	}
1210
406
 
1211
-	ret = i915_gem_object_set_to_gtt_domain(obj, false);
407
+	ret = i915_gem_object_lock_interruptible(obj, NULL);
1212
408
 	if (ret)
1213
409
 		goto out_unpin;
1214
410
 
1215
-	mutex_unlock(&i915->drm.struct_mutex);
411
+	ret = i915_gem_object_set_to_gtt_domain(obj, false);
412
+	if (ret) {
413
+		i915_gem_object_unlock(obj);
414
+		goto out_unpin;
415
+	}
416
+
417
+	fence = i915_gem_object_lock_fence(obj);
418
+	i915_gem_object_unlock(obj);
419
+	if (!fence) {
420
+		ret = -ENOMEM;
421
+		goto out_unpin;
422
+	}
1216
423
 
1217
424
 	user_data = u64_to_user_ptr(args->data_ptr);
1218
425
 	remain = args->size;
..
..
@@ -1229,12 +436,10 @@
1229
436
 		unsigned page_offset = offset_in_page(offset);
1230
437
 		unsigned page_length = PAGE_SIZE - page_offset;
1231
438
 		page_length = remain < page_length ? remain : page_length;
1232
-		if (node.allocated) {
1233
-			wmb();
439
+		if (drm_mm_node_allocated(&node)) {
1234
440
 			ggtt->vm.insert_page(&ggtt->vm,
1235
441
 					     i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
1236
442
 					     node.start, I915_CACHE_NONE, 0);
1237
-			wmb();
1238
443
 		} else {
1239
444
 			page_base += offset & PAGE_MASK;
1240
445
 		}
..
..
@@ -1250,19 +455,16 @@
1250
455
 		offset += page_length;
1251
456
 	}
1252
457
 
1253
-	mutex_lock(&i915->drm.struct_mutex);
458
+	i915_gem_object_unlock_fence(obj, fence);
1254
459
 out_unpin:
1255
-	if (node.allocated) {
1256
-		wmb();
460
+	if (drm_mm_node_allocated(&node)) {
1257
461
 		ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
1258
-		remove_mappable_node(&node);
462
+		remove_mappable_node(ggtt, &node);
1259
463
 	} else {
1260
464
 		i915_vma_unpin(vma);
1261
465
 	}
1262
-out_unlock:
1263
-	intel_runtime_pm_put(i915);
1264
-	mutex_unlock(&i915->drm.struct_mutex);
1265
-
466
+out_rpm:
467
+	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1266
468
 	return ret;
1267
469
 }
1268
470
 
..
..
@@ -1285,8 +487,7 @@
1285
487
 	if (args->size == 0)
1286
488
 		return 0;
1287
489
 
1288
-	if (!access_ok(VERIFY_WRITE,
1289
-		       u64_to_user_ptr(args->data_ptr),
490
+	if (!access_ok(u64_to_user_ptr(args->data_ptr),
1290
491
 		       args->size))
1291
492
 		return -EFAULT;
1292
493
 
..
..
@@ -1302,10 +503,15 @@
1302
503
 
1303
504
 	trace_i915_gem_object_pread(obj, args->offset, args->size);
1304
505
 
506
+	ret = -ENODEV;
507
+	if (obj->ops->pread)
508
+		ret = obj->ops->pread(obj, args);
509
+	if (ret != -ENODEV)
510
+		goto out;
511
+
1305
512
 	ret = i915_gem_object_wait(obj,
1306
513
 				   I915_WAIT_INTERRUPTIBLE,
1307
-				   MAX_SCHEDULE_TIMEOUT,
1308
-				   to_rps_client(file));
514
+				   MAX_SCHEDULE_TIMEOUT);
1309
515
 	if (ret)
1310
516
 		goto out;
1311
517
 
..
..
@@ -1362,15 +568,14 @@
1362
568
 {
1363
569
 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
1364
570
 	struct i915_ggtt *ggtt = &i915->ggtt;
571
+	struct intel_runtime_pm *rpm = &i915->runtime_pm;
572
+	intel_wakeref_t wakeref;
1365
573
 	struct drm_mm_node node;
574
+	struct dma_fence *fence;
1366
575
 	struct i915_vma *vma;
1367
576
 	u64 remain, offset;
1368
577
 	void __user *user_data;
1369
578
 	int ret;
1370
-
1371
-	ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
1372
-	if (ret)
1373
-		return ret;
1374
579
 
1375
580
 	if (i915_gem_object_has_struct_page(obj)) {
1376
581
 		/*
..
..
@@ -1380,42 +585,48 @@
1380
585
 		 * This easily dwarfs any performance advantage from
1381
586
 		 * using the cache bypass of indirect GGTT access.
1382
587
 		 */
1383
-		if (!intel_runtime_pm_get_if_in_use(i915)) {
1384
-			ret = -EFAULT;
1385
-			goto out_unlock;
1386
-		}
588
+		wakeref = intel_runtime_pm_get_if_in_use(rpm);
589
+		if (!wakeref)
590
+			return -EFAULT;
1387
591
 	} else {
1388
592
 		/* No backing pages, no fallback, we must force GGTT access */
1389
-		intel_runtime_pm_get(i915);
593
+		wakeref = intel_runtime_pm_get(rpm);
1390
594
 	}
1391
595
 
1392
-	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
1393
-				       PIN_MAPPABLE |
1394
-				       PIN_NONFAULT |
1395
-				       PIN_NONBLOCK);
596
+	vma = ERR_PTR(-ENODEV);
597
+	if (!i915_gem_object_is_tiled(obj))
598
+		vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
599
+					       PIN_MAPPABLE |
600
+					       PIN_NONBLOCK /* NOWARN */ |
601
+					       PIN_NOEVICT);
1396
602
 	if (!IS_ERR(vma)) {
1397
603
 		node.start = i915_ggtt_offset(vma);
1398
-		node.allocated = false;
1399
-		ret = i915_vma_put_fence(vma);
1400
-		if (ret) {
1401
-			i915_vma_unpin(vma);
1402
-			vma = ERR_PTR(ret);
1403
-		}
1404
-	}
1405
-	if (IS_ERR(vma)) {
604
+		node.flags = 0;
605
+	} else {
1406
606
 		ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
1407
607
 		if (ret)
1408
608
 			goto out_rpm;
1409
-		GEM_BUG_ON(!node.allocated);
609
+		GEM_BUG_ON(!drm_mm_node_allocated(&node));
1410
610
 	}
1411
611
 
1412
-	ret = i915_gem_object_set_to_gtt_domain(obj, true);
612
+	ret = i915_gem_object_lock_interruptible(obj, NULL);
1413
613
 	if (ret)
1414
614
 		goto out_unpin;
1415
615
 
1416
-	mutex_unlock(&i915->drm.struct_mutex);
616
+	ret = i915_gem_object_set_to_gtt_domain(obj, true);
617
+	if (ret) {
618
+		i915_gem_object_unlock(obj);
619
+		goto out_unpin;
620
+	}
1417
621
 
1418
-	intel_fb_obj_invalidate(obj, ORIGIN_CPU);
622
+	fence = i915_gem_object_lock_fence(obj);
623
+	i915_gem_object_unlock(obj);
624
+	if (!fence) {
625
+		ret = -ENOMEM;
626
+		goto out_unpin;
627
+	}
628
+
629
+	i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
1419
630
 
1420
631
 	user_data = u64_to_user_ptr(args->data_ptr);
1421
632
 	offset = args->offset;
..
..
@@ -1431,8 +642,9 @@
1431
642
 		unsigned int page_offset = offset_in_page(offset);
1432
643
 		unsigned int page_length = PAGE_SIZE - page_offset;
1433
644
 		page_length = remain < page_length ? remain : page_length;
1434
-		if (node.allocated) {
1435
-			wmb(); /* flush the write before we modify the GGTT */
645
+		if (drm_mm_node_allocated(&node)) {
646
+			/* flush the write before we modify the GGTT */
647
+			intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1436
648
 			ggtt->vm.insert_page(&ggtt->vm,
1437
649
 					     i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
1438
650
 					     node.start, I915_CACHE_NONE, 0);
..
..
@@ -1456,49 +668,21 @@
1456
668
 		user_data += page_length;
1457
669
 		offset += page_length;
1458
670
 	}
1459
-	intel_fb_obj_flush(obj, ORIGIN_CPU);
1460
671
 
1461
-	mutex_lock(&i915->drm.struct_mutex);
672
+	intel_gt_flush_ggtt_writes(ggtt->vm.gt);
673
+	i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
674
+
675
+	i915_gem_object_unlock_fence(obj, fence);
1462
676
 out_unpin:
1463
-	if (node.allocated) {
1464
-		wmb();
677
+	if (drm_mm_node_allocated(&node)) {
1465
678
 		ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
1466
-		remove_mappable_node(&node);
679
+		remove_mappable_node(ggtt, &node);
1467
680
 	} else {
1468
681
 		i915_vma_unpin(vma);
1469
682
 	}
1470
683
 out_rpm:
1471
-	intel_runtime_pm_put(i915);
1472
-out_unlock:
1473
-	mutex_unlock(&i915->drm.struct_mutex);
684
+	intel_runtime_pm_put(rpm, wakeref);
1474
685
 	return ret;
1475
-}
1476
-
1477
-static int
1478
-shmem_pwrite_slow(struct page *page, int offset, int length,
1479
-		  char __user *user_data,
1480
-		  bool page_do_bit17_swizzling,
1481
-		  bool needs_clflush_before,
1482
-		  bool needs_clflush_after)
1483
-{
1484
-	char *vaddr;
1485
-	int ret;
1486
-
1487
-	vaddr = kmap(page);
1488
-	if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
1489
-		shmem_clflush_swizzled_range(vaddr + offset, length,
1490
-					     page_do_bit17_swizzling);
1491
-	if (page_do_bit17_swizzling)
1492
-		ret = __copy_from_user_swizzled(vaddr, offset, user_data,
1493
-						length);
1494
-	else
1495
-		ret = __copy_from_user(vaddr + offset, user_data, length);
1496
-	if (needs_clflush_after)
1497
-		shmem_clflush_swizzled_range(vaddr + offset, length,
1498
-					     page_do_bit17_swizzling);
1499
-	kunmap(page);
1500
-
1501
-	return ret ? -EFAULT : 0;
1502
686
 }
1503
687
 
1504
688
 /* Per-page copy function for the shmem pwrite fastpath.
..
..
@@ -1508,58 +692,54 @@
1508
692
  */
1509
693
 static int
1510
694
 shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
1511
-	     bool page_do_bit17_swizzling,
1512
695
 	     bool needs_clflush_before,
1513
696
 	     bool needs_clflush_after)
1514
697
 {
698
+	char *vaddr;
1515
699
 	int ret;
1516
700
 
1517
-	ret = -ENODEV;
1518
-	if (!page_do_bit17_swizzling) {
1519
-		char *vaddr = kmap_atomic(page);
701
+	vaddr = kmap(page);
1520
702
 
1521
-		if (needs_clflush_before)
1522
-			drm_clflush_virt_range(vaddr + offset, len);
1523
-		ret = __copy_from_user_inatomic(vaddr + offset, user_data, len);
1524
-		if (needs_clflush_after)
1525
-			drm_clflush_virt_range(vaddr + offset, len);
703
+	if (needs_clflush_before)
704
+		drm_clflush_virt_range(vaddr + offset, len);
1526
705
 
1527
-		kunmap_atomic(vaddr);
1528
-	}
1529
-	if (ret == 0)
1530
-		return ret;
706
+	ret = __copy_from_user(vaddr + offset, user_data, len);
707
+	if (!ret && needs_clflush_after)
708
+		drm_clflush_virt_range(vaddr + offset, len);
1531
709
 
1532
-	return shmem_pwrite_slow(page, offset, len, user_data,
1533
-				 page_do_bit17_swizzling,
1534
-				 needs_clflush_before,
1535
-				 needs_clflush_after);
710
+	kunmap(page);
711
+
712
+	return ret ? -EFAULT : 0;
1536
713
 }
1537
714
 
1538
715
 static int
1539
716
 i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
1540
717
 		      const struct drm_i915_gem_pwrite *args)
1541
718
 {
1542
-	struct drm_i915_private *i915 = to_i915(obj->base.dev);
1543
-	void __user *user_data;
1544
-	u64 remain;
1545
-	unsigned int obj_do_bit17_swizzling;
1546
719
 	unsigned int partial_cacheline_write;
1547
720
 	unsigned int needs_clflush;
1548
721
 	unsigned int offset, idx;
722
+	struct dma_fence *fence;
723
+	void __user *user_data;
724
+	u64 remain;
1549
725
 	int ret;
1550
726
 
1551
-	ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
727
+	ret = i915_gem_object_lock_interruptible(obj, NULL);
1552
728
 	if (ret)
1553
729
 		return ret;
1554
730
 
1555
-	ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
1556
-	mutex_unlock(&i915->drm.struct_mutex);
1557
-	if (ret)
731
+	ret = i915_gem_object_prepare_write(obj, &needs_clflush);
732
+	if (ret) {
733
+		i915_gem_object_unlock(obj);
1558
734
 		return ret;
735
+	}
1559
736
 
1560
-	obj_do_bit17_swizzling = 0;
1561
-	if (i915_gem_object_needs_bit17_swizzle(obj))
1562
-		obj_do_bit17_swizzling = BIT(17);
737
+	fence = i915_gem_object_lock_fence(obj);
738
+	i915_gem_object_finish_access(obj);
739
+	i915_gem_object_unlock(obj);
740
+
741
+	if (!fence)
742
+		return -ENOMEM;
1563
743
 
1564
744
 	/* If we don't overwrite a cacheline completely we need to be
1565
745
 	 * careful to have up-to-date data by first clflushing. Don't
..
..
@@ -1577,7 +757,6 @@
1577
757
 		unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
1578
758
 
1579
759
 		ret = shmem_pwrite(page, offset, length, user_data,
1580
-				   page_to_phys(page) & obj_do_bit17_swizzling,
1581
760
 				   (offset | length) & partial_cacheline_write,
1582
761
 				   needs_clflush & CLFLUSH_AFTER);
1583
762
 		if (ret)
..
..
@@ -1588,8 +767,9 @@
1588
767
 		offset = 0;
1589
768
 	}
1590
769
 
1591
-	intel_fb_obj_flush(obj, ORIGIN_CPU);
1592
-	i915_gem_obj_finish_shmem_access(obj);
770
+	i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
771
+	i915_gem_object_unlock_fence(obj, fence);
772
+
1593
773
 	return ret;
1594
774
 }
1595
775
 
..
..
@@ -1612,9 +792,7 @@
1612
792
 	if (args->size == 0)
1613
793
 		return 0;
1614
794
 
1615
-	if (!access_ok(VERIFY_READ,
1616
-		       u64_to_user_ptr(args->data_ptr),
1617
-		       args->size))
795
+	if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size))
1618
796
 		return -EFAULT;
1619
797
 
1620
798
 	obj = i915_gem_object_lookup(file, args->handle);
..
..
@@ -1644,8 +822,7 @@
1644
822
 	ret = i915_gem_object_wait(obj,
1645
823
 				   I915_WAIT_INTERRUPTIBLE |
1646
824
 				   I915_WAIT_ALL,
1647
-				   MAX_SCHEDULE_TIMEOUT,
1648
-				   to_rps_client(file));
825
+				   MAX_SCHEDULE_TIMEOUT);
1649
826
 	if (ret)
1650
827
 		goto err;
1651
828
 
..
..
@@ -1669,9 +846,7 @@
1669
846
 		ret = i915_gem_gtt_pwrite_fast(obj, args);
1670
847
 
1671
848
 	if (ret == -EFAULT || ret == -ENOSPC) {
1672
-		if (obj->phys_handle)
1673
-			ret = i915_gem_phys_pwrite(obj, args, file);
1674
-		else
849
+		if (i915_gem_object_has_struct_page(obj))
1675
850
 			ret = i915_gem_shmem_pwrite(obj, args);
1676
851
 	}
1677
852
 
..
..
@@ -1679,125 +854,6 @@
1679
854
 err:
1680
855
 	i915_gem_object_put(obj);
1681
856
 	return ret;
1682
-}
1683
-
1684
-static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
1685
-{
1686
-	struct drm_i915_private *i915;
1687
-	struct list_head *list;
1688
-	struct i915_vma *vma;
1689
-
1690
-	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
1691
-
1692
-	for_each_ggtt_vma(vma, obj) {
1693
-		if (i915_vma_is_active(vma))
1694
-			continue;
1695
-
1696
-		if (!drm_mm_node_allocated(&vma->node))
1697
-			continue;
1698
-
1699
-		list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
1700
-	}
1701
-
1702
-	i915 = to_i915(obj->base.dev);
1703
-	spin_lock(&i915->mm.obj_lock);
1704
-	list = obj->bind_count ? &i915->mm.bound_list : &i915->mm.unbound_list;
1705
-	list_move_tail(&obj->mm.link, list);
1706
-	spin_unlock(&i915->mm.obj_lock);
1707
-}
1708
-
1709
-/**
1710
- * Called when user space prepares to use an object with the CPU, either
1711
- * through the mmap ioctl's mapping or a GTT mapping.
1712
- * @dev: drm device
1713
- * @data: ioctl data blob
1714
- * @file: drm file
1715
- */
1716
-int
1717
-i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1718
-			  struct drm_file *file)
1719
-{
1720
-	struct drm_i915_gem_set_domain *args = data;
1721
-	struct drm_i915_gem_object *obj;
1722
-	uint32_t read_domains = args->read_domains;
1723
-	uint32_t write_domain = args->write_domain;
1724
-	int err;
1725
-
1726
-	/* Only handle setting domains to types used by the CPU. */
1727
-	if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
1728
-		return -EINVAL;
1729
-
1730
-	/* Having something in the write domain implies it's in the read
1731
-	 * domain, and only that read domain.  Enforce that in the request.
1732
-	 */
1733
-	if (write_domain != 0 && read_domains != write_domain)
1734
-		return -EINVAL;
1735
-
1736
-	obj = i915_gem_object_lookup(file, args->handle);
1737
-	if (!obj)
1738
-		return -ENOENT;
1739
-
1740
-	/* Try to flush the object off the GPU without holding the lock.
1741
-	 * We will repeat the flush holding the lock in the normal manner
1742
-	 * to catch cases where we are gazumped.
1743
-	 */
1744
-	err = i915_gem_object_wait(obj,
1745
-				   I915_WAIT_INTERRUPTIBLE |
1746
-				   (write_domain ? I915_WAIT_ALL : 0),
1747
-				   MAX_SCHEDULE_TIMEOUT,
1748
-				   to_rps_client(file));
1749
-	if (err)
1750
-		goto out;
1751
-
1752
-	/*
1753
-	 * Proxy objects do not control access to the backing storage, ergo
1754
-	 * they cannot be used as a means to manipulate the cache domain
1755
-	 * tracking for that backing storage. The proxy object is always
1756
-	 * considered to be outside of any cache domain.
1757
-	 */
1758
-	if (i915_gem_object_is_proxy(obj)) {
1759
-		err = -ENXIO;
1760
-		goto out;
1761
-	}
1762
-
1763
-	/*
1764
-	 * Flush and acquire obj->pages so that we are coherent through
1765
-	 * direct access in memory with previous cached writes through
1766
-	 * shmemfs and that our cache domain tracking remains valid.
1767
-	 * For example, if the obj->filp was moved to swap without us
1768
-	 * being notified and releasing the pages, we would mistakenly
1769
-	 * continue to assume that the obj remained out of the CPU cached
1770
-	 * domain.
1771
-	 */
1772
-	err = i915_gem_object_pin_pages(obj);
1773
-	if (err)
1774
-		goto out;
1775
-
1776
-	err = i915_mutex_lock_interruptible(dev);
1777
-	if (err)
1778
-		goto out_unpin;
1779
-
1780
-	if (read_domains & I915_GEM_DOMAIN_WC)
1781
-		err = i915_gem_object_set_to_wc_domain(obj, write_domain);
1782
-	else if (read_domains & I915_GEM_DOMAIN_GTT)
1783
-		err = i915_gem_object_set_to_gtt_domain(obj, write_domain);
1784
-	else
1785
-		err = i915_gem_object_set_to_cpu_domain(obj, write_domain);
1786
-
1787
-	/* And bump the LRU for this access */
1788
-	i915_gem_object_bump_inactive_ggtt(obj);
1789
-
1790
-	mutex_unlock(&dev->struct_mutex);
1791
-
1792
-	if (write_domain != 0)
1793
-		intel_fb_obj_invalidate(obj,
1794
-					fb_write_origin(obj, write_domain));
1795
-
1796
-out_unpin:
1797
-	i915_gem_object_unpin_pages(obj);
1798
-out:
1799
-	i915_gem_object_put(obj);
1800
-	return err;
1801
857
 }
1802
858
 
1803
859
 /**
..
..
@@ -1829,424 +885,7 @@
1829
885
 	return 0;
1830
886
 }
1831
887
 
1832
-static inline bool
1833
-__vma_matches(struct vm_area_struct *vma, struct file *filp,
1834
-	      unsigned long addr, unsigned long size)
1835
-{
1836
-	if (vma->vm_file != filp)
1837
-		return false;
1838
-
1839
-	return vma->vm_start == addr &&
1840
-	       (vma->vm_end - vma->vm_start) == PAGE_ALIGN(size);
1841
-}
1842
-
1843
-/**
1844
- * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
1845
- *			 it is mapped to.
1846
- * @dev: drm device
1847
- * @data: ioctl data blob
1848
- * @file: drm file
1849
- *
1850
- * While the mapping holds a reference on the contents of the object, it doesn't
1851
- * imply a ref on the object itself.
1852
- *
1853
- * IMPORTANT:
1854
- *
1855
- * DRM driver writers who look a this function as an example for how to do GEM
1856
- * mmap support, please don't implement mmap support like here. The modern way
1857
- * to implement DRM mmap support is with an mmap offset ioctl (like
1858
- * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1859
- * That way debug tooling like valgrind will understand what's going on, hiding
1860
- * the mmap call in a driver private ioctl will break that. The i915 driver only
1861
- * does cpu mmaps this way because we didn't know better.
1862
- */
1863
-int
1864
-i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1865
-		    struct drm_file *file)
1866
-{
1867
-	struct drm_i915_gem_mmap *args = data;
1868
-	struct drm_i915_gem_object *obj;
1869
-	unsigned long addr;
1870
-
1871
-	if (args->flags & ~(I915_MMAP_WC))
1872
-		return -EINVAL;
1873
-
1874
-	if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
1875
-		return -ENODEV;
1876
-
1877
-	obj = i915_gem_object_lookup(file, args->handle);
1878
-	if (!obj)
1879
-		return -ENOENT;
1880
-
1881
-	/* prime objects have no backing filp to GEM mmap
1882
-	 * pages from.
1883
-	 */
1884
-	if (!obj->base.filp) {
1885
-		addr = -ENXIO;
1886
-		goto err;
1887
-	}
1888
-
1889
-	if (range_overflows(args->offset, args->size, (u64)obj->base.size)) {
1890
-		addr = -EINVAL;
1891
-		goto err;
1892
-	}
1893
-
1894
-	addr = vm_mmap(obj->base.filp, 0, args->size,
1895
-		       PROT_READ | PROT_WRITE, MAP_SHARED,
1896
-		       args->offset);
1897
-	if (IS_ERR_VALUE(addr))
1898
-		goto err;
1899
-
1900
-	if (args->flags & I915_MMAP_WC) {
1901
-		struct mm_struct *mm = current->mm;
1902
-		struct vm_area_struct *vma;
1903
-
1904
-		if (down_write_killable(&mm->mmap_sem)) {
1905
-			addr = -EINTR;
1906
-			goto err;
1907
-		}
1908
-		vma = find_vma(mm, addr);
1909
-		if (vma && __vma_matches(vma, obj->base.filp, addr, args->size))
1910
-			vma->vm_page_prot =
1911
-				pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1912
-		else
1913
-			addr = -ENOMEM;
1914
-		up_write(&mm->mmap_sem);
1915
-		if (IS_ERR_VALUE(addr))
1916
-			goto err;
1917
-
1918
-		/* This may race, but that's ok, it only gets set */
1919
-		WRITE_ONCE(obj->frontbuffer_ggtt_origin, ORIGIN_CPU);
1920
-	}
1921
-	i915_gem_object_put(obj);
1922
-
1923
-	args->addr_ptr = (uint64_t) addr;
1924
-	return 0;
1925
-
1926
-err:
1927
-	i915_gem_object_put(obj);
1928
-	return addr;
1929
-}
1930
-
1931
-static unsigned int tile_row_pages(struct drm_i915_gem_object *obj)
1932
-{
1933
-	return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT;
1934
-}
1935
-
1936
-/**
1937
- * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps
1938
- *
1939
- * A history of the GTT mmap interface:
1940
- *
1941
- * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to
1942
- *     aligned and suitable for fencing, and still fit into the available
1943
- *     mappable space left by the pinned display objects. A classic problem
1944
- *     we called the page-fault-of-doom where we would ping-pong between
1945
- *     two objects that could not fit inside the GTT and so the memcpy
1946
- *     would page one object in at the expense of the other between every
1947
- *     single byte.
1948
- *
1949
- * 1 - Objects can be any size, and have any compatible fencing (X Y, or none
1950
- *     as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the
1951
- *     object is too large for the available space (or simply too large
1952
- *     for the mappable aperture!), a view is created instead and faulted
1953
- *     into userspace. (This view is aligned and sized appropriately for
1954
- *     fenced access.)
1955
- *
1956
- * 2 - Recognise WC as a separate cache domain so that we can flush the
1957
- *     delayed writes via GTT before performing direct access via WC.
1958
- *
1959
- * Restrictions:
1960
- *
1961
- *  * snoopable objects cannot be accessed via the GTT. It can cause machine
1962
- *    hangs on some architectures, corruption on others. An attempt to service
1963
- *    a GTT page fault from a snoopable object will generate a SIGBUS.
1964
- *
1965
- *  * the object must be able to fit into RAM (physical memory, though no
1966
- *    limited to the mappable aperture).
1967
- *
1968
- *
1969
- * Caveats:
1970
- *
1971
- *  * a new GTT page fault will synchronize rendering from the GPU and flush
1972
- *    all data to system memory. Subsequent access will not be synchronized.
1973
- *
1974
- *  * all mappings are revoked on runtime device suspend.
1975
- *
1976
- *  * there are only 8, 16 or 32 fence registers to share between all users
1977
- *    (older machines require fence register for display and blitter access
1978
- *    as well). Contention of the fence registers will cause the previous users
1979
- *    to be unmapped and any new access will generate new page faults.
1980
- *
1981
- *  * running out of memory while servicing a fault may generate a SIGBUS,
1982
- *    rather than the expected SIGSEGV.
1983
- */
1984
-int i915_gem_mmap_gtt_version(void)
1985
-{
1986
-	return 2;
1987
-}
1988
-
1989
-static inline struct i915_ggtt_view
1990
-compute_partial_view(struct drm_i915_gem_object *obj,
1991
-		     pgoff_t page_offset,
1992
-		     unsigned int chunk)
1993
-{
1994
-	struct i915_ggtt_view view;
1995
-
1996
-	if (i915_gem_object_is_tiled(obj))
1997
-		chunk = roundup(chunk, tile_row_pages(obj));
1998
-
1999
-	view.type = I915_GGTT_VIEW_PARTIAL;
2000
-	view.partial.offset = rounddown(page_offset, chunk);
2001
-	view.partial.size =
2002
-		min_t(unsigned int, chunk,
2003
-		      (obj->base.size >> PAGE_SHIFT) - view.partial.offset);
2004
-
2005
-	/* If the partial covers the entire object, just create a normal VMA. */
2006
-	if (chunk >= obj->base.size >> PAGE_SHIFT)
2007
-		view.type = I915_GGTT_VIEW_NORMAL;
2008
-
2009
-	return view;
2010
-}
2011
-
2012
-/**
2013
- * i915_gem_fault - fault a page into the GTT
2014
- * @vmf: fault info
2015
- *
2016
- * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
2017
- * from userspace.  The fault handler takes care of binding the object to
2018
- * the GTT (if needed), allocating and programming a fence register (again,
2019
- * only if needed based on whether the old reg is still valid or the object
2020
- * is tiled) and inserting a new PTE into the faulting process.
2021
- *
2022
- * Note that the faulting process may involve evicting existing objects
2023
- * from the GTT and/or fence registers to make room.  So performance may
2024
- * suffer if the GTT working set is large or there are few fence registers
2025
- * left.
2026
- *
2027
- * The current feature set supported by i915_gem_fault() and thus GTT mmaps
2028
- * is exposed via I915_PARAM_MMAP_GTT_VERSION (see i915_gem_mmap_gtt_version).
2029
- */
2030
-vm_fault_t i915_gem_fault(struct vm_fault *vmf)
2031
-{
2032
-#define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT)
2033
-	struct vm_area_struct *area = vmf->vma;
2034
-	struct drm_i915_gem_object *obj = to_intel_bo(area->vm_private_data);
2035
-	struct drm_device *dev = obj->base.dev;
2036
-	struct drm_i915_private *dev_priv = to_i915(dev);
2037
-	struct i915_ggtt *ggtt = &dev_priv->ggtt;
2038
-	bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
2039
-	struct i915_vma *vma;
2040
-	pgoff_t page_offset;
2041
-	int ret;
2042
-
2043
-	/* Sanity check that we allow writing into this object */
2044
-	if (i915_gem_object_is_readonly(obj) && write)
2045
-		return VM_FAULT_SIGBUS;
2046
-
2047
-	/* We don't use vmf->pgoff since that has the fake offset */
2048
-	page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT;
2049
-
2050
-	trace_i915_gem_object_fault(obj, page_offset, true, write);
2051
-
2052
-	/* Try to flush the object off the GPU first without holding the lock.
2053
-	 * Upon acquiring the lock, we will perform our sanity checks and then
2054
-	 * repeat the flush holding the lock in the normal manner to catch cases
2055
-	 * where we are gazumped.
2056
-	 */
2057
-	ret = i915_gem_object_wait(obj,
2058
-				   I915_WAIT_INTERRUPTIBLE,
2059
-				   MAX_SCHEDULE_TIMEOUT,
2060
-				   NULL);
2061
-	if (ret)
2062
-		goto err;
2063
-
2064
-	ret = i915_gem_object_pin_pages(obj);
2065
-	if (ret)
2066
-		goto err;
2067
-
2068
-	intel_runtime_pm_get(dev_priv);
2069
-
2070
-	ret = i915_mutex_lock_interruptible(dev);
2071
-	if (ret)
2072
-		goto err_rpm;
2073
-
2074
-	/* Access to snoopable pages through the GTT is incoherent. */
2075
-	if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev_priv)) {
2076
-		ret = -EFAULT;
2077
-		goto err_unlock;
2078
-	}
2079
-
2080
-
2081
-	/* Now pin it into the GTT as needed */
2082
-	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
2083
-				       PIN_MAPPABLE |
2084
-				       PIN_NONBLOCK |
2085
-				       PIN_NONFAULT);
2086
-	if (IS_ERR(vma)) {
2087
-		/* Use a partial view if it is bigger than available space */
2088
-		struct i915_ggtt_view view =
2089
-			compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES);
2090
-		unsigned int flags;
2091
-
2092
-		flags = PIN_MAPPABLE;
2093
-		if (view.type == I915_GGTT_VIEW_NORMAL)
2094
-			flags |= PIN_NONBLOCK; /* avoid warnings for pinned */
2095
-
2096
-		/*
2097
-		 * Userspace is now writing through an untracked VMA, abandon
2098
-		 * all hope that the hardware is able to track future writes.
2099
-		 */
2100
-		obj->frontbuffer_ggtt_origin = ORIGIN_CPU;
2101
-
2102
-		vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags);
2103
-		if (IS_ERR(vma) && !view.type) {
2104
-			flags = PIN_MAPPABLE;
2105
-			view.type = I915_GGTT_VIEW_PARTIAL;
2106
-			vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags);
2107
-		}
2108
-	}
2109
-	if (IS_ERR(vma)) {
2110
-		ret = PTR_ERR(vma);
2111
-		goto err_unlock;
2112
-	}
2113
-
2114
-	ret = i915_gem_object_set_to_gtt_domain(obj, write);
2115
-	if (ret)
2116
-		goto err_unpin;
2117
-
2118
-	ret = i915_vma_pin_fence(vma);
2119
-	if (ret)
2120
-		goto err_unpin;
2121
-
2122
-	/* Finally, remap it using the new GTT offset */
2123
-	ret = remap_io_mapping(area,
2124
-			       area->vm_start + (vma->ggtt_view.partial.offset << PAGE_SHIFT),
2125
-			       (ggtt->gmadr.start + vma->node.start) >> PAGE_SHIFT,
2126
-			       min_t(u64, vma->size, area->vm_end - area->vm_start),
2127
-			       &ggtt->iomap);
2128
-	if (ret)
2129
-		goto err_fence;
2130
-
2131
-	/* Mark as being mmapped into userspace for later revocation */
2132
-	assert_rpm_wakelock_held(dev_priv);
2133
-	if (!i915_vma_set_userfault(vma) && !obj->userfault_count++)
2134
-		list_add(&obj->userfault_link, &dev_priv->mm.userfault_list);
2135
-	GEM_BUG_ON(!obj->userfault_count);
2136
-
2137
-	i915_vma_set_ggtt_write(vma);
2138
-
2139
-err_fence:
2140
-	i915_vma_unpin_fence(vma);
2141
-err_unpin:
2142
-	__i915_vma_unpin(vma);
2143
-err_unlock:
2144
-	mutex_unlock(&dev->struct_mutex);
2145
-err_rpm:
2146
-	intel_runtime_pm_put(dev_priv);
2147
-	i915_gem_object_unpin_pages(obj);
2148
-err:
2149
-	switch (ret) {
2150
-	case -EIO:
2151
-		/*
2152
-		 * We eat errors when the gpu is terminally wedged to avoid
2153
-		 * userspace unduly crashing (gl has no provisions for mmaps to
2154
-		 * fail). But any other -EIO isn't ours (e.g. swap in failure)
2155
-		 * and so needs to be reported.
2156
-		 */
2157
-		if (!i915_terminally_wedged(&dev_priv->gpu_error))
2158
-			return VM_FAULT_SIGBUS;
2159
-		/* else: fall through */
2160
-	case -EAGAIN:
2161
-		/*
2162
-		 * EAGAIN means the gpu is hung and we'll wait for the error
2163
-		 * handler to reset everything when re-faulting in
2164
-		 * i915_mutex_lock_interruptible.
2165
-		 */
2166
-	case 0:
2167
-	case -ERESTARTSYS:
2168
-	case -EINTR:
2169
-	case -EBUSY:
2170
-		/*
2171
-		 * EBUSY is ok: this just means that another thread
2172
-		 * already did the job.
2173
-		 */
2174
-		return VM_FAULT_NOPAGE;
2175
-	case -ENOMEM:
2176
-		return VM_FAULT_OOM;
2177
-	case -ENOSPC:
2178
-	case -EFAULT:
2179
-		return VM_FAULT_SIGBUS;
2180
-	default:
2181
-		WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
2182
-		return VM_FAULT_SIGBUS;
2183
-	}
2184
-}
2185
-
2186
-static void __i915_gem_object_release_mmap(struct drm_i915_gem_object *obj)
2187
-{
2188
-	struct i915_vma *vma;
2189
-
2190
-	GEM_BUG_ON(!obj->userfault_count);
2191
-
2192
-	obj->userfault_count = 0;
2193
-	list_del(&obj->userfault_link);
2194
-	drm_vma_node_unmap(&obj->base.vma_node,
2195
-			   obj->base.dev->anon_inode->i_mapping);
2196
-
2197
-	for_each_ggtt_vma(vma, obj)
2198
-		i915_vma_unset_userfault(vma);
2199
-}
2200
-
2201
-/**
2202
- * i915_gem_release_mmap - remove physical page mappings
2203
- * @obj: obj in question
2204
- *
2205
- * Preserve the reservation of the mmapping with the DRM core code, but
2206
- * relinquish ownership of the pages back to the system.
2207
- *
2208
- * It is vital that we remove the page mapping if we have mapped a tiled
2209
- * object through the GTT and then lose the fence register due to
2210
- * resource pressure. Similarly if the object has been moved out of the
2211
- * aperture, than pages mapped into userspace must be revoked. Removing the
2212
- * mapping will then trigger a page fault on the next user access, allowing
2213
- * fixup by i915_gem_fault().
2214
- */
2215
-void
2216
-i915_gem_release_mmap(struct drm_i915_gem_object *obj)
2217
-{
2218
-	struct drm_i915_private *i915 = to_i915(obj->base.dev);
2219
-
2220
-	/* Serialisation between user GTT access and our code depends upon
2221
-	 * revoking the CPU's PTE whilst the mutex is held. The next user
2222
-	 * pagefault then has to wait until we release the mutex.
2223
-	 *
2224
-	 * Note that RPM complicates somewhat by adding an additional
2225
-	 * requirement that operations to the GGTT be made holding the RPM
2226
-	 * wakeref.
2227
-	 */
2228
-	lockdep_assert_held(&i915->drm.struct_mutex);
2229
-	intel_runtime_pm_get(i915);
2230
-
2231
-	if (!obj->userfault_count)
2232
-		goto out;
2233
-
2234
-	__i915_gem_object_release_mmap(obj);
2235
-
2236
-	/* Ensure that the CPU's PTE are revoked and there are not outstanding
2237
-	 * memory transactions from userspace before we return. The TLB
2238
-	 * flushing implied above by changing the PTE above *should* be
2239
-	 * sufficient, an extra barrier here just provides us with a bit
2240
-	 * of paranoid documentation about our requirement to serialise
2241
-	 * memory writes before touching registers / GSM.
2242
-	 */
2243
-	wmb();
2244
-
2245
-out:
2246
-	intel_runtime_pm_put(i915);
2247
-}
2248
-
2249
-void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv)
888
+void i915_gem_runtime_suspend(struct drm_i915_private *i915)
2250
889
 {
2251
890
 	struct drm_i915_gem_object *obj, *on;
2252
891
 	int i;
..
..
@@ -2259,17 +898,19 @@
2259
898
 	 */
2260
899
 
2261
900
 	list_for_each_entry_safe(obj, on,
2262
-				 &dev_priv->mm.userfault_list, userfault_link)
2263
-		__i915_gem_object_release_mmap(obj);
901
+				 &i915->ggtt.userfault_list, userfault_link)
902
+		__i915_gem_object_release_mmap_gtt(obj);
2264
903
 
2265
-	/* The fence will be lost when the device powers down. If any were
904
+	/*
905
+	 * The fence will be lost when the device powers down. If any were
2266
906
 	 * in use by hardware (i.e. they are pinned), we should not be powering
2267
907
 	 * down! All other fences will be reacquired by the user upon waking.
2268
908
 	 */
2269
-	for (i = 0; i < dev_priv->num_fence_regs; i++) {
2270
-		struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
909
+	for (i = 0; i < i915->ggtt.num_fences; i++) {
910
+		struct i915_fence_reg *reg = &i915->ggtt.fence_regs[i];
2271
911
 
2272
-		/* Ideally we want to assert that the fence register is not
912
+		/*
913
+		 * Ideally we want to assert that the fence register is not
2273
914
 		 * live at this point (i.e. that no piece of code will be
2274
915
 		 * trying to write through fence + GTT, as that both violates
2275
916
 		 * our tracking of activity and associated locking/barriers,
..
..
@@ -2288,2287 +929,44 @@
2288
929
 	}
2289
930
 }
2290
931
 
2291
-static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
932
+static void discard_ggtt_vma(struct i915_vma *vma)
2292
933
 {
2293
-	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2294
-	int err;
934
+	struct drm_i915_gem_object *obj = vma->obj;
2295
935
 
2296
-	err = drm_gem_create_mmap_offset(&obj->base);
2297
-	if (likely(!err))
2298
-		return 0;
2299
-
2300
-	/* Attempt to reap some mmap space from dead objects */
2301
-	do {
2302
-		err = i915_gem_wait_for_idle(dev_priv,
2303
-					     I915_WAIT_INTERRUPTIBLE,
2304
-					     MAX_SCHEDULE_TIMEOUT);
2305
-		if (err)
2306
-			break;
2307
-
2308
-		i915_gem_drain_freed_objects(dev_priv);
2309
-		err = drm_gem_create_mmap_offset(&obj->base);
2310
-		if (!err)
2311
-			break;
2312
-
2313
-	} while (flush_delayed_work(&dev_priv->gt.retire_work));
2314
-
2315
-	return err;
2316
-}
2317
-
2318
-static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
2319
-{
2320
-	drm_gem_free_mmap_offset(&obj->base);
2321
-}
2322
-
2323
-int
2324
-i915_gem_mmap_gtt(struct drm_file *file,
2325
-		  struct drm_device *dev,
2326
-		  uint32_t handle,
2327
-		  uint64_t *offset)
2328
-{
2329
-	struct drm_i915_gem_object *obj;
2330
-	int ret;
2331
-
2332
-	obj = i915_gem_object_lookup(file, handle);
2333
-	if (!obj)
2334
-		return -ENOENT;
2335
-
2336
-	ret = i915_gem_object_create_mmap_offset(obj);
2337
-	if (ret == 0)
2338
-		*offset = drm_vma_node_offset_addr(&obj->base.vma_node);
2339
-
2340
-	i915_gem_object_put(obj);
2341
-	return ret;
2342
-}
2343
-
2344
-/**
2345
- * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
2346
- * @dev: DRM device
2347
- * @data: GTT mapping ioctl data
2348
- * @file: GEM object info
2349
- *
2350
- * Simply returns the fake offset to userspace so it can mmap it.
2351
- * The mmap call will end up in drm_gem_mmap(), which will set things
2352
- * up so we can get faults in the handler above.
2353
- *
2354
- * The fault handler will take care of binding the object into the GTT
2355
- * (since it may have been evicted to make room for something), allocating
2356
- * a fence register, and mapping the appropriate aperture address into
2357
- * userspace.
2358
- */
2359
-int
2360
-i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
2361
-			struct drm_file *file)
2362
-{
2363
-	struct drm_i915_gem_mmap_gtt *args = data;
2364
-
2365
-	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
2366
-}
2367
-
2368
-/* Immediately discard the backing storage */
2369
-static void
2370
-i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2371
-{
2372
-	i915_gem_object_free_mmap_offset(obj);
2373
-
2374
-	if (obj->base.filp == NULL)
2375
-		return;
2376
-
2377
-	/* Our goal here is to return as much of the memory as
2378
-	 * is possible back to the system as we are called from OOM.
2379
-	 * To do this we must instruct the shmfs to drop all of its
2380
-	 * backing pages, *now*.
2381
-	 */
2382
-	shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
2383
-	obj->mm.madv = __I915_MADV_PURGED;
2384
-	obj->mm.pages = ERR_PTR(-EFAULT);
2385
-}
2386
-
2387
-/* Try to discard unwanted pages */
2388
-void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
2389
-{
2390
-	struct address_space *mapping;
2391
-
2392
-	lockdep_assert_held(&obj->mm.lock);
2393
-	GEM_BUG_ON(i915_gem_object_has_pages(obj));
2394
-
2395
-	switch (obj->mm.madv) {
2396
-	case I915_MADV_DONTNEED:
2397
-		i915_gem_object_truncate(obj);
2398
-	case __I915_MADV_PURGED:
2399
-		return;
936
+	spin_lock(&obj->vma.lock);
937
+	if (!RB_EMPTY_NODE(&vma->obj_node)) {
938
+		rb_erase(&vma->obj_node, &obj->vma.tree);
939
+		RB_CLEAR_NODE(&vma->obj_node);
2400
940
 	}
2401
-
2402
-	if (obj->base.filp == NULL)
2403
-		return;
2404
-
2405
-	mapping = obj->base.filp->f_mapping,
2406
-	invalidate_mapping_pages(mapping, 0, (loff_t)-1);
941
+	spin_unlock(&obj->vma.lock);
2407
942
 }
2408
943
 
2409
-/*
2410
- * Move pages to appropriate lru and release the pagevec, decrementing the
2411
- * ref count of those pages.
2412
- */
2413
-static void check_release_pagevec(struct pagevec *pvec)
2414
-{
2415
-	check_move_unevictable_pages(pvec);
2416
-	__pagevec_release(pvec);
2417
-	cond_resched();
2418
-}
2419
-
2420
-static void
2421
-i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj,
2422
-			      struct sg_table *pages)
2423
-{
2424
-	struct sgt_iter sgt_iter;
2425
-	struct pagevec pvec;
2426
-	struct page *page;
2427
-
2428
-	__i915_gem_object_release_shmem(obj, pages, true);
2429
-
2430
-	i915_gem_gtt_finish_pages(obj, pages);
2431
-
2432
-	if (i915_gem_object_needs_bit17_swizzle(obj))
2433
-		i915_gem_object_save_bit_17_swizzle(obj, pages);
2434
-
2435
-	mapping_clear_unevictable(file_inode(obj->base.filp)->i_mapping);
2436
-
2437
-	pagevec_init(&pvec);
2438
-	for_each_sgt_page(page, sgt_iter, pages) {
2439
-		if (obj->mm.dirty)
2440
-			set_page_dirty(page);
2441
-
2442
-		if (obj->mm.madv == I915_MADV_WILLNEED)
2443
-			mark_page_accessed(page);
2444
-
2445
-		put_page(page);
2446
-	}
2447
-	obj->mm.dirty = false;
2448
-
2449
-	sg_free_table(pages);
2450
-	kfree(pages);
2451
-}
2452
-
2453
-static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
2454
-{
2455
-	struct radix_tree_iter iter;
2456
-	void __rcu **slot;
2457
-
2458
-	rcu_read_lock();
2459
-	radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
2460
-		radix_tree_delete(&obj->mm.get_page.radix, iter.index);
2461
-	rcu_read_unlock();
2462
-}
2463
-
2464
-struct reg_and_bit {
2465
-	i915_reg_t reg;
2466
-	u32 bit;
2467
-};
2468
-
2469
-static struct reg_and_bit
2470
-get_reg_and_bit(const struct intel_engine_cs *engine,
2471
-		const i915_reg_t *regs, const unsigned int num)
2472
-{
2473
-	const unsigned int class = engine->class;
2474
-	struct reg_and_bit rb = { .bit = 1 };
2475
-
2476
-	if (WARN_ON_ONCE(class >= num || !regs[class].reg))
2477
-		return rb;
2478
-
2479
-	rb.reg = regs[class];
2480
-	if (class == VIDEO_DECODE_CLASS)
2481
-		rb.reg.reg += 4 * engine->instance; /* GEN8_M2TCR */
2482
-
2483
-	return rb;
2484
-}
2485
-
2486
-static void invalidate_tlbs(struct drm_i915_private *dev_priv)
2487
-{
2488
-	static const i915_reg_t gen8_regs[] = {
2489
-		[RENDER_CLASS]                  = GEN8_RTCR,
2490
-		[VIDEO_DECODE_CLASS]            = GEN8_M1TCR, /* , GEN8_M2TCR */
2491
-		[VIDEO_ENHANCEMENT_CLASS]       = GEN8_VTCR,
2492
-		[COPY_ENGINE_CLASS]             = GEN8_BTCR,
2493
-	};
2494
-	const unsigned int num = ARRAY_SIZE(gen8_regs);
2495
-	const i915_reg_t *regs = gen8_regs;
2496
-	struct intel_engine_cs *engine;
2497
-	enum intel_engine_id id;
2498
-
2499
-	if (INTEL_GEN(dev_priv) < 8)
2500
-		return;
2501
-
2502
-	GEM_TRACE("\n");
2503
-
2504
-	assert_rpm_wakelock_held(dev_priv);
2505
-
2506
-	mutex_lock(&dev_priv->tlb_invalidate_lock);
2507
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
2508
-
2509
-	for_each_engine(engine, dev_priv, id) {
2510
-		/*
2511
-		 * HW architecture suggest typical invalidation time at 40us,
2512
-		 * with pessimistic cases up to 100us and a recommendation to
2513
-		 * cap at 1ms. We go a bit higher just in case.
2514
-		 */
2515
-		const unsigned int timeout_us = 100;
2516
-		const unsigned int timeout_ms = 4;
2517
-		struct reg_and_bit rb;
2518
-
2519
-		rb = get_reg_and_bit(engine, regs, num);
2520
-		if (!i915_mmio_reg_offset(rb.reg))
2521
-			continue;
2522
-
2523
-		I915_WRITE_FW(rb.reg, rb.bit);
2524
-		if (__intel_wait_for_register_fw(dev_priv,
2525
-						 rb.reg, rb.bit, 0,
2526
-						 timeout_us, timeout_ms,
2527
-						 NULL))
2528
-			DRM_ERROR_RATELIMITED("%s TLB invalidation did not complete in %ums!\n",
2529
-					      engine->name, timeout_ms);
2530
-	}
2531
-
2532
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
2533
-	mutex_unlock(&dev_priv->tlb_invalidate_lock);
2534
-}
2535
-
2536
-static struct sg_table *
2537
-__i915_gem_object_unset_pages(struct drm_i915_gem_object *obj)
944
+struct i915_vma *
945
+i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj,
946
+			    struct i915_gem_ww_ctx *ww,
947
+			    const struct i915_ggtt_view *view,
948
+			    u64 size, u64 alignment, u64 flags)
2538
949
 {
2539
950
 	struct drm_i915_private *i915 = to_i915(obj->base.dev);
2540
-	struct sg_table *pages;
2541
-
2542
-	pages = fetch_and_zero(&obj->mm.pages);
2543
-	if (!pages)
2544
-		return NULL;
2545
-
2546
-	spin_lock(&i915->mm.obj_lock);
2547
-	list_del(&obj->mm.link);
2548
-	spin_unlock(&i915->mm.obj_lock);
2549
-
2550
-	if (obj->mm.mapping) {
2551
-		void *ptr;
2552
-
2553
-		ptr = page_mask_bits(obj->mm.mapping);
2554
-		if (is_vmalloc_addr(ptr))
2555
-			vunmap(ptr);
2556
-		else
2557
-			kunmap(kmap_to_page(ptr));
2558
-
2559
-		obj->mm.mapping = NULL;
2560
-	}
2561
-
2562
-	__i915_gem_object_reset_page_iter(obj);
2563
-	obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;
2564
-
2565
-	if (test_and_clear_bit(I915_BO_WAS_BOUND_BIT, &obj->flags)) {
2566
-		struct drm_i915_private *i915 = to_i915(obj->base.dev);
2567
-
2568
-		if (intel_runtime_pm_get_if_in_use(i915)) {
2569
-			invalidate_tlbs(i915);
2570
-			intel_runtime_pm_put(i915);
2571
-		}
2572
-	}
2573
-
2574
-	return pages;
2575
-}
2576
-
2577
-void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj,
2578
-				 enum i915_mm_subclass subclass)
2579
-{
2580
-	struct sg_table *pages;
2581
-
2582
-	if (i915_gem_object_has_pinned_pages(obj))
2583
-		return;
2584
-
2585
-	GEM_BUG_ON(obj->bind_count);
2586
-	if (!i915_gem_object_has_pages(obj))
2587
-		return;
2588
-
2589
-	/* May be called by shrinker from within get_pages() (on another bo) */
2590
-	mutex_lock_nested(&obj->mm.lock, subclass);
2591
-	if (unlikely(atomic_read(&obj->mm.pages_pin_count)))
2592
-		goto unlock;
2593
-
2594
-	/*
2595
-	 * ->put_pages might need to allocate memory for the bit17 swizzle
2596
-	 * array, hence protect them from being reaped by removing them from gtt
2597
-	 * lists early.
2598
-	 */
2599
-	pages = __i915_gem_object_unset_pages(obj);
2600
-	if (!IS_ERR(pages))
2601
-		obj->ops->put_pages(obj, pages);
2602
-
2603
-unlock:
2604
-	mutex_unlock(&obj->mm.lock);
2605
-}
2606
-
2607
-static bool i915_sg_trim(struct sg_table *orig_st)
2608
-{
2609
-	struct sg_table new_st;
2610
-	struct scatterlist *sg, *new_sg;
2611
-	unsigned int i;
2612
-
2613
-	if (orig_st->nents == orig_st->orig_nents)
2614
-		return false;
2615
-
2616
-	if (sg_alloc_table(&new_st, orig_st->nents, GFP_KERNEL | __GFP_NOWARN))
2617
-		return false;
2618
-
2619
-	new_sg = new_st.sgl;
2620
-	for_each_sg(orig_st->sgl, sg, orig_st->nents, i) {
2621
-		sg_set_page(new_sg, sg_page(sg), sg->length, 0);
2622
-		/* called before being DMA mapped, no need to copy sg->dma_* */
2623
-		new_sg = sg_next(new_sg);
2624
-	}
2625
-	GEM_BUG_ON(new_sg); /* Should walk exactly nents and hit the end */
2626
-
2627
-	sg_free_table(orig_st);
2628
-
2629
-	*orig_st = new_st;
2630
-	return true;
2631
-}
2632
-
2633
-static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2634
-{
2635
-	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
2636
-	const unsigned long page_count = obj->base.size / PAGE_SIZE;
2637
-	unsigned long i;
2638
-	struct address_space *mapping;
2639
-	struct sg_table *st;
2640
-	struct scatterlist *sg;
2641
-	struct sgt_iter sgt_iter;
2642
-	struct page *page;
2643
-	unsigned long last_pfn = 0;	/* suppress gcc warning */
2644
-	unsigned int max_segment = i915_sg_segment_size();
2645
-	unsigned int sg_page_sizes;
2646
-	struct pagevec pvec;
2647
-	gfp_t noreclaim;
2648
-	int ret;
2649
-
2650
-	/* Assert that the object is not currently in any GPU domain. As it
2651
-	 * wasn't in the GTT, there shouldn't be any way it could have been in
2652
-	 * a GPU cache
2653
-	 */
2654
-	GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2655
-	GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
2656
-
2657
-	st = kmalloc(sizeof(*st), GFP_KERNEL);
2658
-	if (st == NULL)
2659
-		return -ENOMEM;
2660
-
2661
-rebuild_st:
2662
-	if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2663
-		kfree(st);
2664
-		return -ENOMEM;
2665
-	}
2666
-
2667
-	/* Get the list of pages out of our struct file.  They'll be pinned
2668
-	 * at this point until we release them.
2669
-	 *
2670
-	 * Fail silently without starting the shrinker
2671
-	 */
2672
-	mapping = obj->base.filp->f_mapping;
2673
-	mapping_set_unevictable(mapping);
2674
-	noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM);
2675
-	noreclaim |= __GFP_NORETRY | __GFP_NOWARN;
2676
-
2677
-	sg = st->sgl;
2678
-	st->nents = 0;
2679
-	sg_page_sizes = 0;
2680
-	for (i = 0; i < page_count; i++) {
2681
-		const unsigned int shrink[] = {
2682
-			I915_SHRINK_BOUND | I915_SHRINK_UNBOUND | I915_SHRINK_PURGEABLE,
2683
-			0,
2684
-		}, *s = shrink;
2685
-		gfp_t gfp = noreclaim;
2686
-
2687
-		do {
2688
-			page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2689
-			if (likely(!IS_ERR(page)))
2690
-				break;
2691
-
2692
-			if (!*s) {
2693
-				ret = PTR_ERR(page);
2694
-				goto err_sg;
2695
-			}
2696
-
2697
-			i915_gem_shrink(dev_priv, 2 * page_count, NULL, *s++);
2698
-			cond_resched();
2699
-
2700
-			/* We've tried hard to allocate the memory by reaping
2701
-			 * our own buffer, now let the real VM do its job and
2702
-			 * go down in flames if truly OOM.
2703
-			 *
2704
-			 * However, since graphics tend to be disposable,
2705
-			 * defer the oom here by reporting the ENOMEM back
2706
-			 * to userspace.
2707
-			 */
2708
-			if (!*s) {
2709
-				/* reclaim and warn, but no oom */
2710
-				gfp = mapping_gfp_mask(mapping);
2711
-
2712
-				/* Our bo are always dirty and so we require
2713
-				 * kswapd to reclaim our pages (direct reclaim
2714
-				 * does not effectively begin pageout of our
2715
-				 * buffers on its own). However, direct reclaim
2716
-				 * only waits for kswapd when under allocation
2717
-				 * congestion. So as a result __GFP_RECLAIM is
2718
-				 * unreliable and fails to actually reclaim our
2719
-				 * dirty pages -- unless you try over and over
2720
-				 * again with !__GFP_NORETRY. However, we still
2721
-				 * want to fail this allocation rather than
2722
-				 * trigger the out-of-memory killer and for
2723
-				 * this we want __GFP_RETRY_MAYFAIL.
2724
-				 */
2725
-				gfp |= __GFP_RETRY_MAYFAIL;
2726
-			}
2727
-		} while (1);
2728
-
2729
-		if (!i ||
2730
-		    sg->length >= max_segment ||
2731
-		    page_to_pfn(page) != last_pfn + 1) {
2732
-			if (i) {
2733
-				sg_page_sizes |= sg->length;
2734
-				sg = sg_next(sg);
2735
-			}
2736
-			st->nents++;
2737
-			sg_set_page(sg, page, PAGE_SIZE, 0);
2738
-		} else {
2739
-			sg->length += PAGE_SIZE;
2740
-		}
2741
-		last_pfn = page_to_pfn(page);
2742
-
2743
-		/* Check that the i965g/gm workaround works. */
2744
-		WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2745
-	}
2746
-	if (sg) { /* loop terminated early; short sg table */
2747
-		sg_page_sizes |= sg->length;
2748
-		sg_mark_end(sg);
2749
-	}
2750
-
2751
-	/* Trim unused sg entries to avoid wasting memory. */
2752
-	i915_sg_trim(st);
2753
-
2754
-	ret = i915_gem_gtt_prepare_pages(obj, st);
2755
-	if (ret) {
2756
-		/* DMA remapping failed? One possible cause is that
2757
-		 * it could not reserve enough large entries, asking
2758
-		 * for PAGE_SIZE chunks instead may be helpful.
2759
-		 */
2760
-		if (max_segment > PAGE_SIZE) {
2761
-			for_each_sgt_page(page, sgt_iter, st)
2762
-				put_page(page);
2763
-			sg_free_table(st);
2764
-
2765
-			max_segment = PAGE_SIZE;
2766
-			goto rebuild_st;
2767
-		} else {
2768
-			dev_warn(&dev_priv->drm.pdev->dev,
2769
-				 "Failed to DMA remap %lu pages\n",
2770
-				 page_count);
2771
-			goto err_pages;
2772
-		}
2773
-	}
2774
-
2775
-	if (i915_gem_object_needs_bit17_swizzle(obj))
2776
-		i915_gem_object_do_bit_17_swizzle(obj, st);
2777
-
2778
-	__i915_gem_object_set_pages(obj, st, sg_page_sizes);
2779
-
2780
-	return 0;
2781
-
2782
-err_sg:
2783
-	sg_mark_end(sg);
2784
-err_pages:
2785
-	mapping_clear_unevictable(mapping);
2786
-	pagevec_init(&pvec);
2787
-	for_each_sgt_page(page, sgt_iter, st) {
2788
-		if (!pagevec_add(&pvec, page))
2789
-			check_release_pagevec(&pvec);
2790
-	}
2791
-	if (pagevec_count(&pvec))
2792
-		check_release_pagevec(&pvec);
2793
-	sg_free_table(st);
2794
-	kfree(st);
2795
-
2796
-	/* shmemfs first checks if there is enough memory to allocate the page
2797
-	 * and reports ENOSPC should there be insufficient, along with the usual
2798
-	 * ENOMEM for a genuine allocation failure.
2799
-	 *
2800
-	 * We use ENOSPC in our driver to mean that we have run out of aperture
2801
-	 * space and so want to translate the error from shmemfs back to our
2802
-	 * usual understanding of ENOMEM.
2803
-	 */
2804
-	if (ret == -ENOSPC)
2805
-		ret = -ENOMEM;
2806
-
2807
-	return ret;
2808
-}
2809
-
2810
-void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
2811
-				 struct sg_table *pages,
2812
-				 unsigned int sg_page_sizes)
2813
-{
2814
-	struct drm_i915_private *i915 = to_i915(obj->base.dev);
2815
-	unsigned long supported = INTEL_INFO(i915)->page_sizes;
2816
-	int i;
2817
-
2818
-	lockdep_assert_held(&obj->mm.lock);
2819
-
2820
-	obj->mm.get_page.sg_pos = pages->sgl;
2821
-	obj->mm.get_page.sg_idx = 0;
2822
-
2823
-	obj->mm.pages = pages;
2824
-
2825
-	if (i915_gem_object_is_tiled(obj) &&
2826
-	    i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
2827
-		GEM_BUG_ON(obj->mm.quirked);
2828
-		__i915_gem_object_pin_pages(obj);
2829
-		obj->mm.quirked = true;
2830
-	}
2831
-
2832
-	GEM_BUG_ON(!sg_page_sizes);
2833
-	obj->mm.page_sizes.phys = sg_page_sizes;
2834
-
2835
-	/*
2836
-	 * Calculate the supported page-sizes which fit into the given
2837
-	 * sg_page_sizes. This will give us the page-sizes which we may be able
2838
-	 * to use opportunistically when later inserting into the GTT. For
2839
-	 * example if phys=2G, then in theory we should be able to use 1G, 2M,
2840
-	 * 64K or 4K pages, although in practice this will depend on a number of
2841
-	 * other factors.
2842
-	 */
2843
-	obj->mm.page_sizes.sg = 0;
2844
-	for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
2845
-		if (obj->mm.page_sizes.phys & ~0u << i)
2846
-			obj->mm.page_sizes.sg |= BIT(i);
2847
-	}
2848
-	GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg));
2849
-
2850
-	spin_lock(&i915->mm.obj_lock);
2851
-	list_add(&obj->mm.link, &i915->mm.unbound_list);
2852
-	spin_unlock(&i915->mm.obj_lock);
2853
-}
2854
-
2855
-static int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2856
-{
2857
-	int err;
2858
-
2859
-	if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
2860
-		DRM_DEBUG("Attempting to obtain a purgeable object\n");
2861
-		return -EFAULT;
2862
-	}
2863
-
2864
-	err = obj->ops->get_pages(obj);
2865
-	GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj));
2866
-
2867
-	return err;
2868
-}
2869
-
2870
-/* Ensure that the associated pages are gathered from the backing storage
2871
- * and pinned into our object. i915_gem_object_pin_pages() may be called
2872
- * multiple times before they are released by a single call to
2873
- * i915_gem_object_unpin_pages() - once the pages are no longer referenced
2874
- * either as a result of memory pressure (reaping pages under the shrinker)
2875
- * or as the object is itself released.
2876
- */
2877
-int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2878
-{
2879
-	int err;
2880
-
2881
-	err = mutex_lock_interruptible(&obj->mm.lock);
2882
-	if (err)
2883
-		return err;
2884
-
2885
-	if (unlikely(!i915_gem_object_has_pages(obj))) {
2886
-		GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
2887
-
2888
-		err = ____i915_gem_object_get_pages(obj);
2889
-		if (err)
2890
-			goto unlock;
2891
-
2892
-		smp_mb__before_atomic();
2893
-	}
2894
-	atomic_inc(&obj->mm.pages_pin_count);
2895
-
2896
-unlock:
2897
-	mutex_unlock(&obj->mm.lock);
2898
-	return err;
2899
-}
2900
-
2901
-/* The 'mapping' part of i915_gem_object_pin_map() below */
2902
-static void *i915_gem_object_map(const struct drm_i915_gem_object *obj,
2903
-				 enum i915_map_type type)
2904
-{
2905
-	unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
2906
-	struct sg_table *sgt = obj->mm.pages;
2907
-	struct sgt_iter sgt_iter;
2908
-	struct page *page;
2909
-	struct page *stack_pages[32];
2910
-	struct page **pages = stack_pages;
2911
-	unsigned long i = 0;
2912
-	pgprot_t pgprot;
2913
-	void *addr;
2914
-
2915
-	/* A single page can always be kmapped */
2916
-	if (n_pages == 1 && type == I915_MAP_WB)
2917
-		return kmap(sg_page(sgt->sgl));
2918
-
2919
-	if (n_pages > ARRAY_SIZE(stack_pages)) {
2920
-		/* Too big for stack -- allocate temporary array instead */
2921
-		pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL);
2922
-		if (!pages)
2923
-			return NULL;
2924
-	}
2925
-
2926
-	for_each_sgt_page(page, sgt_iter, sgt)
2927
-		pages[i++] = page;
2928
-
2929
-	/* Check that we have the expected number of pages */
2930
-	GEM_BUG_ON(i != n_pages);
2931
-
2932
-	switch (type) {
2933
-	default:
2934
-		MISSING_CASE(type);
2935
-		/* fallthrough to use PAGE_KERNEL anyway */
2936
-	case I915_MAP_WB:
2937
-		pgprot = PAGE_KERNEL;
2938
-		break;
2939
-	case I915_MAP_WC:
2940
-		pgprot = pgprot_writecombine(PAGE_KERNEL_IO);
2941
-		break;
2942
-	}
2943
-	addr = vmap(pages, n_pages, 0, pgprot);
2944
-
2945
-	if (pages != stack_pages)
2946
-		kvfree(pages);
2947
-
2948
-	return addr;
2949
-}
2950
-
2951
-/* get, pin, and map the pages of the object into kernel space */
2952
-void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
2953
-			      enum i915_map_type type)
2954
-{
2955
-	enum i915_map_type has_type;
2956
-	bool pinned;
2957
-	void *ptr;
2958
-	int ret;
2959
-
2960
-	if (unlikely(!i915_gem_object_has_struct_page(obj)))
2961
-		return ERR_PTR(-ENXIO);
2962
-
2963
-	ret = mutex_lock_interruptible(&obj->mm.lock);
2964
-	if (ret)
2965
-		return ERR_PTR(ret);
2966
-
2967
-	pinned = !(type & I915_MAP_OVERRIDE);
2968
-	type &= ~I915_MAP_OVERRIDE;
2969
-
2970
-	if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) {
2971
-		if (unlikely(!i915_gem_object_has_pages(obj))) {
2972
-			GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
2973
-
2974
-			ret = ____i915_gem_object_get_pages(obj);
2975
-			if (ret)
2976
-				goto err_unlock;
2977
-
2978
-			smp_mb__before_atomic();
2979
-		}
2980
-		atomic_inc(&obj->mm.pages_pin_count);
2981
-		pinned = false;
2982
-	}
2983
-	GEM_BUG_ON(!i915_gem_object_has_pages(obj));
2984
-
2985
-	ptr = page_unpack_bits(obj->mm.mapping, &has_type);
2986
-	if (ptr && has_type != type) {
2987
-		if (pinned) {
2988
-			ret = -EBUSY;
2989
-			goto err_unpin;
2990
-		}
2991
-
2992
-		if (is_vmalloc_addr(ptr))
2993
-			vunmap(ptr);
2994
-		else
2995
-			kunmap(kmap_to_page(ptr));
2996
-
2997
-		ptr = obj->mm.mapping = NULL;
2998
-	}
2999
-
3000
-	if (!ptr) {
3001
-		ptr = i915_gem_object_map(obj, type);
3002
-		if (!ptr) {
3003
-			ret = -ENOMEM;
3004
-			goto err_unpin;
3005
-		}
3006
-
3007
-		obj->mm.mapping = page_pack_bits(ptr, type);
3008
-	}
3009
-
3010
-out_unlock:
3011
-	mutex_unlock(&obj->mm.lock);
3012
-	return ptr;
3013
-
3014
-err_unpin:
3015
-	atomic_dec(&obj->mm.pages_pin_count);
3016
-err_unlock:
3017
-	ptr = ERR_PTR(ret);
3018
-	goto out_unlock;
3019
-}
3020
-
3021
-static int
3022
-i915_gem_object_pwrite_gtt(struct drm_i915_gem_object *obj,
3023
-			   const struct drm_i915_gem_pwrite *arg)
3024
-{
3025
-	struct address_space *mapping = obj->base.filp->f_mapping;
3026
-	char __user *user_data = u64_to_user_ptr(arg->data_ptr);
3027
-	u64 remain, offset;
3028
-	unsigned int pg;
3029
-
3030
-	/* Before we instantiate/pin the backing store for our use, we
3031
-	 * can prepopulate the shmemfs filp efficiently using a write into
3032
-	 * the pagecache. We avoid the penalty of instantiating all the
3033
-	 * pages, important if the user is just writing to a few and never
3034
-	 * uses the object on the GPU, and using a direct write into shmemfs
3035
-	 * allows it to avoid the cost of retrieving a page (either swapin
3036
-	 * or clearing-before-use) before it is overwritten.
3037
-	 */
3038
-	if (i915_gem_object_has_pages(obj))
3039
-		return -ENODEV;
3040
-
3041
-	if (obj->mm.madv != I915_MADV_WILLNEED)
3042
-		return -EFAULT;
3043
-
3044
-	/* Before the pages are instantiated the object is treated as being
3045
-	 * in the CPU domain. The pages will be clflushed as required before
3046
-	 * use, and we can freely write into the pages directly. If userspace
3047
-	 * races pwrite with any other operation; corruption will ensue -
3048
-	 * that is userspace's prerogative!
3049
-	 */
3050
-
3051
-	remain = arg->size;
3052
-	offset = arg->offset;
3053
-	pg = offset_in_page(offset);
3054
-
3055
-	do {
3056
-		unsigned int len, unwritten;
3057
-		struct page *page;
3058
-		void *data, *vaddr;
3059
-		int err;
3060
-
3061
-		len = PAGE_SIZE - pg;
3062
-		if (len > remain)
3063
-			len = remain;
3064
-
3065
-		err = pagecache_write_begin(obj->base.filp, mapping,
3066
-					    offset, len, 0,
3067
-					    &page, &data);
3068
-		if (err < 0)
3069
-			return err;
3070
-
3071
-		vaddr = kmap(page);
3072
-		unwritten = copy_from_user(vaddr + pg, user_data, len);
3073
-		kunmap(page);
3074
-
3075
-		err = pagecache_write_end(obj->base.filp, mapping,
3076
-					  offset, len, len - unwritten,
3077
-					  page, data);
3078
-		if (err < 0)
3079
-			return err;
3080
-
3081
-		if (unwritten)
3082
-			return -EFAULT;
3083
-
3084
-		remain -= len;
3085
-		user_data += len;
3086
-		offset += len;
3087
-		pg = 0;
3088
-	} while (remain);
3089
-
3090
-	return 0;
3091
-}
3092
-
3093
-static void i915_gem_client_mark_guilty(struct drm_i915_file_private *file_priv,
3094
-					const struct i915_gem_context *ctx)
3095
-{
3096
-	unsigned int score;
3097
-	unsigned long prev_hang;
3098
-
3099
-	if (i915_gem_context_is_banned(ctx))
3100
-		score = I915_CLIENT_SCORE_CONTEXT_BAN;
3101
-	else
3102
-		score = 0;
3103
-
3104
-	prev_hang = xchg(&file_priv->hang_timestamp, jiffies);
3105
-	if (time_before(jiffies, prev_hang + I915_CLIENT_FAST_HANG_JIFFIES))
3106
-		score += I915_CLIENT_SCORE_HANG_FAST;
3107
-
3108
-	if (score) {
3109
-		atomic_add(score, &file_priv->ban_score);
3110
-
3111
-		DRM_DEBUG_DRIVER("client %s: gained %u ban score, now %u\n",
3112
-				 ctx->name, score,
3113
-				 atomic_read(&file_priv->ban_score));
3114
-	}
3115
-}
3116
-
3117
-static void i915_gem_context_mark_guilty(struct i915_gem_context *ctx)
3118
-{
3119
-	unsigned int score;
3120
-	bool banned, bannable;
3121
-
3122
-	atomic_inc(&ctx->guilty_count);
3123
-
3124
-	bannable = i915_gem_context_is_bannable(ctx);
3125
-	score = atomic_add_return(CONTEXT_SCORE_GUILTY, &ctx->ban_score);
3126
-	banned = score >= CONTEXT_SCORE_BAN_THRESHOLD;
3127
-
3128
-	/* Cool contexts don't accumulate client ban score */
3129
-	if (!bannable)
3130
-		return;
3131
-
3132
-	if (banned) {
3133
-		DRM_DEBUG_DRIVER("context %s: guilty %d, score %u, banned\n",
3134
-				 ctx->name, atomic_read(&ctx->guilty_count),
3135
-				 score);
3136
-		i915_gem_context_set_banned(ctx);
3137
-	}
3138
-
3139
-	if (!IS_ERR_OR_NULL(ctx->file_priv))
3140
-		i915_gem_client_mark_guilty(ctx->file_priv, ctx);
3141
-}
3142
-
3143
-static void i915_gem_context_mark_innocent(struct i915_gem_context *ctx)
3144
-{
3145
-	atomic_inc(&ctx->active_count);
3146
-}
3147
-
3148
-struct i915_request *
3149
-i915_gem_find_active_request(struct intel_engine_cs *engine)
3150
-{
3151
-	struct i915_request *request, *active = NULL;
3152
-	unsigned long flags;
3153
-
3154
-	/*
3155
-	 * We are called by the error capture, reset and to dump engine
3156
-	 * state at random points in time. In particular, note that neither is
3157
-	 * crucially ordered with an interrupt. After a hang, the GPU is dead
3158
-	 * and we assume that no more writes can happen (we waited long enough
3159
-	 * for all writes that were in transaction to be flushed) - adding an
3160
-	 * extra delay for a recent interrupt is pointless. Hence, we do
3161
-	 * not need an engine->irq_seqno_barrier() before the seqno reads.
3162
-	 * At all other times, we must assume the GPU is still running, but
3163
-	 * we only care about the snapshot of this moment.
3164
-	 */
3165
-	spin_lock_irqsave(&engine->timeline.lock, flags);
3166
-	list_for_each_entry(request, &engine->timeline.requests, link) {
3167
-		if (__i915_request_completed(request, request->global_seqno))
3168
-			continue;
3169
-
3170
-		active = request;
3171
-		break;
3172
-	}
3173
-	spin_unlock_irqrestore(&engine->timeline.lock, flags);
3174
-
3175
-	return active;
3176
-}
3177
-
3178
-/*
3179
- * Ensure irq handler finishes, and not run again.
3180
- * Also return the active request so that we only search for it once.
3181
- */
3182
-struct i915_request *
3183
-i915_gem_reset_prepare_engine(struct intel_engine_cs *engine)
3184
-{
3185
-	struct i915_request *request;
3186
-
3187
-	/*
3188
-	 * During the reset sequence, we must prevent the engine from
3189
-	 * entering RC6. As the context state is undefined until we restart
3190
-	 * the engine, if it does enter RC6 during the reset, the state
3191
-	 * written to the powercontext is undefined and so we may lose
3192
-	 * GPU state upon resume, i.e. fail to restart after a reset.
3193
-	 */
3194
-	intel_uncore_forcewake_get(engine->i915, FORCEWAKE_ALL);
3195
-
3196
-	request = engine->reset.prepare(engine);
3197
-	if (request && request->fence.error == -EIO)
3198
-		request = ERR_PTR(-EIO); /* Previous reset failed! */
3199
-
3200
-	return request;
3201
-}
3202
-
3203
-int i915_gem_reset_prepare(struct drm_i915_private *dev_priv)
3204
-{
3205
-	struct intel_engine_cs *engine;
3206
-	struct i915_request *request;
3207
-	enum intel_engine_id id;
3208
-	int err = 0;
3209
-
3210
-	for_each_engine(engine, dev_priv, id) {
3211
-		request = i915_gem_reset_prepare_engine(engine);
3212
-		if (IS_ERR(request)) {
3213
-			err = PTR_ERR(request);
3214
-			continue;
3215
-		}
3216
-
3217
-		engine->hangcheck.active_request = request;
3218
-	}
3219
-
3220
-	i915_gem_revoke_fences(dev_priv);
3221
-	intel_uc_sanitize(dev_priv);
3222
-
3223
-	return err;
3224
-}
3225
-
3226
-static void engine_skip_context(struct i915_request *request)
3227
-{
3228
-	struct intel_engine_cs *engine = request->engine;
3229
-	struct i915_gem_context *hung_ctx = request->gem_context;
3230
-	struct i915_timeline *timeline = request->timeline;
3231
-	unsigned long flags;
3232
-
3233
-	GEM_BUG_ON(timeline == &engine->timeline);
3234
-
3235
-	spin_lock_irqsave(&engine->timeline.lock, flags);
3236
-	spin_lock(&timeline->lock);
3237
-
3238
-	list_for_each_entry_continue(request, &engine->timeline.requests, link)
3239
-		if (request->gem_context == hung_ctx)
3240
-			i915_request_skip(request, -EIO);
3241
-
3242
-	list_for_each_entry(request, &timeline->requests, link)
3243
-		i915_request_skip(request, -EIO);
3244
-
3245
-	spin_unlock(&timeline->lock);
3246
-	spin_unlock_irqrestore(&engine->timeline.lock, flags);
3247
-}
3248
-
3249
-/* Returns the request if it was guilty of the hang */
3250
-static struct i915_request *
3251
-i915_gem_reset_request(struct intel_engine_cs *engine,
3252
-		       struct i915_request *request,
3253
-		       bool stalled)
3254
-{
3255
-	/* The guilty request will get skipped on a hung engine.
3256
-	 *
3257
-	 * Users of client default contexts do not rely on logical
3258
-	 * state preserved between batches so it is safe to execute
3259
-	 * queued requests following the hang. Non default contexts
3260
-	 * rely on preserved state, so skipping a batch loses the
3261
-	 * evolution of the state and it needs to be considered corrupted.
3262
-	 * Executing more queued batches on top of corrupted state is
3263
-	 * risky. But we take the risk by trying to advance through
3264
-	 * the queued requests in order to make the client behaviour
3265
-	 * more predictable around resets, by not throwing away random
3266
-	 * amount of batches it has prepared for execution. Sophisticated
3267
-	 * clients can use gem_reset_stats_ioctl and dma fence status
3268
-	 * (exported via sync_file info ioctl on explicit fences) to observe
3269
-	 * when it loses the context state and should rebuild accordingly.
3270
-	 *
3271
-	 * The context ban, and ultimately the client ban, mechanism are safety
3272
-	 * valves if client submission ends up resulting in nothing more than
3273
-	 * subsequent hangs.
3274
-	 */
3275
-
3276
-	if (i915_request_completed(request)) {
3277
-		GEM_TRACE("%s pardoned global=%d (fence %llx:%d), current %d\n",
3278
-			  engine->name, request->global_seqno,
3279
-			  request->fence.context, request->fence.seqno,
3280
-			  intel_engine_get_seqno(engine));
3281
-		stalled = false;
3282
-	}
3283
-
3284
-	if (stalled) {
3285
-		i915_gem_context_mark_guilty(request->gem_context);
3286
-		i915_request_skip(request, -EIO);
3287
-
3288
-		/* If this context is now banned, skip all pending requests. */
3289
-		if (i915_gem_context_is_banned(request->gem_context))
3290
-			engine_skip_context(request);
3291
-	} else {
3292
-		/*
3293
-		 * Since this is not the hung engine, it may have advanced
3294
-		 * since the hang declaration. Double check by refinding
3295
-		 * the active request at the time of the reset.
3296
-		 */
3297
-		request = i915_gem_find_active_request(engine);
3298
-		if (request) {
3299
-			unsigned long flags;
3300
-
3301
-			i915_gem_context_mark_innocent(request->gem_context);
3302
-			dma_fence_set_error(&request->fence, -EAGAIN);
3303
-
3304
-			/* Rewind the engine to replay the incomplete rq */
3305
-			spin_lock_irqsave(&engine->timeline.lock, flags);
3306
-			request = list_prev_entry(request, link);
3307
-			if (&request->link == &engine->timeline.requests)
3308
-				request = NULL;
3309
-			spin_unlock_irqrestore(&engine->timeline.lock, flags);
3310
-		}
3311
-	}
3312
-
3313
-	return request;
3314
-}
3315
-
3316
-void i915_gem_reset_engine(struct intel_engine_cs *engine,
3317
-			   struct i915_request *request,
3318
-			   bool stalled)
3319
-{
3320
-	/*
3321
-	 * Make sure this write is visible before we re-enable the interrupt
3322
-	 * handlers on another CPU, as tasklet_enable() resolves to just
3323
-	 * a compiler barrier which is insufficient for our purpose here.
3324
-	 */
3325
-	smp_store_mb(engine->irq_posted, 0);
3326
-
3327
-	if (request)
3328
-		request = i915_gem_reset_request(engine, request, stalled);
3329
-
3330
-	/* Setup the CS to resume from the breadcrumb of the hung request */
3331
-	engine->reset.reset(engine, request);
3332
-}
3333
-
3334
-void i915_gem_reset(struct drm_i915_private *dev_priv,
3335
-		    unsigned int stalled_mask)
3336
-{
3337
-	struct intel_engine_cs *engine;
3338
-	enum intel_engine_id id;
3339
-
3340
-	lockdep_assert_held(&dev_priv->drm.struct_mutex);
3341
-
3342
-	i915_retire_requests(dev_priv);
3343
-
3344
-	for_each_engine(engine, dev_priv, id) {
3345
-		struct intel_context *ce;
3346
-
3347
-		i915_gem_reset_engine(engine,
3348
-				      engine->hangcheck.active_request,
3349
-				      stalled_mask & ENGINE_MASK(id));
3350
-		ce = fetch_and_zero(&engine->last_retired_context);
3351
-		if (ce)
3352
-			intel_context_unpin(ce);
3353
-
3354
-		/*
3355
-		 * Ostensibily, we always want a context loaded for powersaving,
3356
-		 * so if the engine is idle after the reset, send a request
3357
-		 * to load our scratch kernel_context.
3358
-		 *
3359
-		 * More mysteriously, if we leave the engine idle after a reset,
3360
-		 * the next userspace batch may hang, with what appears to be
3361
-		 * an incoherent read by the CS (presumably stale TLB). An
3362
-		 * empty request appears sufficient to paper over the glitch.
3363
-		 */
3364
-		if (intel_engine_is_idle(engine)) {
3365
-			struct i915_request *rq;
3366
-
3367
-			rq = i915_request_alloc(engine,
3368
-						dev_priv->kernel_context);
3369
-			if (!IS_ERR(rq))
3370
-				i915_request_add(rq);
3371
-		}
3372
-	}
3373
-
3374
-	i915_gem_restore_fences(dev_priv);
3375
-}
3376
-
3377
-void i915_gem_reset_finish_engine(struct intel_engine_cs *engine)
3378
-{
3379
-	engine->reset.finish(engine);
3380
-
3381
-	intel_uncore_forcewake_put(engine->i915, FORCEWAKE_ALL);
3382
-}
3383
-
3384
-void i915_gem_reset_finish(struct drm_i915_private *dev_priv)
3385
-{
3386
-	struct intel_engine_cs *engine;
3387
-	enum intel_engine_id id;
3388
-
3389
-	lockdep_assert_held(&dev_priv->drm.struct_mutex);
3390
-
3391
-	for_each_engine(engine, dev_priv, id) {
3392
-		engine->hangcheck.active_request = NULL;
3393
-		i915_gem_reset_finish_engine(engine);
3394
-	}
3395
-}
3396
-
3397
-static void nop_submit_request(struct i915_request *request)
3398
-{
3399
-	GEM_TRACE("%s fence %llx:%d -> -EIO\n",
3400
-		  request->engine->name,
3401
-		  request->fence.context, request->fence.seqno);
3402
-	dma_fence_set_error(&request->fence, -EIO);
3403
-
3404
-	i915_request_submit(request);
3405
-}
3406
-
3407
-static void nop_complete_submit_request(struct i915_request *request)
3408
-{
3409
-	unsigned long flags;
3410
-
3411
-	GEM_TRACE("%s fence %llx:%d -> -EIO\n",
3412
-		  request->engine->name,
3413
-		  request->fence.context, request->fence.seqno);
3414
-	dma_fence_set_error(&request->fence, -EIO);
3415
-
3416
-	spin_lock_irqsave(&request->engine->timeline.lock, flags);
3417
-	__i915_request_submit(request);
3418
-	intel_engine_init_global_seqno(request->engine, request->global_seqno);
3419
-	spin_unlock_irqrestore(&request->engine->timeline.lock, flags);
3420
-}
3421
-
3422
-void i915_gem_set_wedged(struct drm_i915_private *i915)
3423
-{
3424
-	struct intel_engine_cs *engine;
3425
-	enum intel_engine_id id;
3426
-
3427
-	GEM_TRACE("start\n");
3428
-
3429
-	if (GEM_SHOW_DEBUG()) {
3430
-		struct drm_printer p = drm_debug_printer(__func__);
3431
-
3432
-		for_each_engine(engine, i915, id)
3433
-			intel_engine_dump(engine, &p, "%s\n", engine->name);
3434
-	}
3435
-
3436
-	set_bit(I915_WEDGED, &i915->gpu_error.flags);
3437
-	smp_mb__after_atomic();
3438
-
3439
-	/*
3440
-	 * First, stop submission to hw, but do not yet complete requests by
3441
-	 * rolling the global seqno forward (since this would complete requests
3442
-	 * for which we haven't set the fence error to EIO yet).
3443
-	 */
3444
-	for_each_engine(engine, i915, id) {
3445
-		i915_gem_reset_prepare_engine(engine);
3446
-
3447
-		engine->submit_request = nop_submit_request;
3448
-		engine->schedule = NULL;
3449
-	}
3450
-	i915->caps.scheduler = 0;
3451
-
3452
-	/* Even if the GPU reset fails, it should still stop the engines */
3453
-	intel_gpu_reset(i915, ALL_ENGINES);
3454
-
3455
-	/*
3456
-	 * Make sure no one is running the old callback before we proceed with
3457
-	 * cancelling requests and resetting the completion tracking. Otherwise
3458
-	 * we might submit a request to the hardware which never completes.
3459
-	 */
3460
-	synchronize_rcu();
3461
-
3462
-	for_each_engine(engine, i915, id) {
3463
-		/* Mark all executing requests as skipped */
3464
-		engine->cancel_requests(engine);
3465
-
3466
-		/*
3467
-		 * Only once we've force-cancelled all in-flight requests can we
3468
-		 * start to complete all requests.
3469
-		 */
3470
-		engine->submit_request = nop_complete_submit_request;
3471
-	}
3472
-
3473
-	/*
3474
-	 * Make sure no request can slip through without getting completed by
3475
-	 * either this call here to intel_engine_init_global_seqno, or the one
3476
-	 * in nop_complete_submit_request.
3477
-	 */
3478
-	synchronize_rcu();
3479
-
3480
-	for_each_engine(engine, i915, id) {
3481
-		unsigned long flags;
3482
-
3483
-		/*
3484
-		 * Mark all pending requests as complete so that any concurrent
3485
-		 * (lockless) lookup doesn't try and wait upon the request as we
3486
-		 * reset it.
3487
-		 */
3488
-		spin_lock_irqsave(&engine->timeline.lock, flags);
3489
-		intel_engine_init_global_seqno(engine,
3490
-					       intel_engine_last_submit(engine));
3491
-		spin_unlock_irqrestore(&engine->timeline.lock, flags);
3492
-
3493
-		i915_gem_reset_finish_engine(engine);
3494
-	}
3495
-
3496
-	GEM_TRACE("end\n");
3497
-
3498
-	wake_up_all(&i915->gpu_error.reset_queue);
3499
-}
3500
-
3501
-bool i915_gem_unset_wedged(struct drm_i915_private *i915)
3502
-{
3503
-	struct i915_timeline *tl;
3504
-
3505
-	lockdep_assert_held(&i915->drm.struct_mutex);
3506
-	if (!test_bit(I915_WEDGED, &i915->gpu_error.flags))
3507
-		return true;
3508
-
3509
-	GEM_TRACE("start\n");
3510
-
3511
-	/*
3512
-	 * Before unwedging, make sure that all pending operations
3513
-	 * are flushed and errored out - we may have requests waiting upon
3514
-	 * third party fences. We marked all inflight requests as EIO, and
3515
-	 * every execbuf since returned EIO, for consistency we want all
3516
-	 * the currently pending requests to also be marked as EIO, which
3517
-	 * is done inside our nop_submit_request - and so we must wait.
3518
-	 *
3519
-	 * No more can be submitted until we reset the wedged bit.
3520
-	 */
3521
-	list_for_each_entry(tl, &i915->gt.timelines, link) {
3522
-		struct i915_request *rq;
3523
-
3524
-		rq = i915_gem_active_peek(&tl->last_request,
3525
-					  &i915->drm.struct_mutex);
3526
-		if (!rq)
3527
-			continue;
3528
-
3529
-		/*
3530
-		 * We can't use our normal waiter as we want to
3531
-		 * avoid recursively trying to handle the current
3532
-		 * reset. The basic dma_fence_default_wait() installs
3533
-		 * a callback for dma_fence_signal(), which is
3534
-		 * triggered by our nop handler (indirectly, the
3535
-		 * callback enables the signaler thread which is
3536
-		 * woken by the nop_submit_request() advancing the seqno
3537
-		 * and when the seqno passes the fence, the signaler
3538
-		 * then signals the fence waking us up).
3539
-		 */
3540
-		if (dma_fence_default_wait(&rq->fence, true,
3541
-					   MAX_SCHEDULE_TIMEOUT) < 0)
3542
-			return false;
3543
-	}
3544
-	i915_retire_requests(i915);
3545
-	GEM_BUG_ON(i915->gt.active_requests);
3546
-
3547
-	/*
3548
-	 * Undo nop_submit_request. We prevent all new i915 requests from
3549
-	 * being queued (by disallowing execbuf whilst wedged) so having
3550
-	 * waited for all active requests above, we know the system is idle
3551
-	 * and do not have to worry about a thread being inside
3552
-	 * engine->submit_request() as we swap over. So unlike installing
3553
-	 * the nop_submit_request on reset, we can do this from normal
3554
-	 * context and do not require stop_machine().
3555
-	 */
3556
-	intel_engines_reset_default_submission(i915);
3557
-	i915_gem_contexts_lost(i915);
3558
-
3559
-	GEM_TRACE("end\n");
3560
-
3561
-	smp_mb__before_atomic(); /* complete takeover before enabling execbuf */
3562
-	clear_bit(I915_WEDGED, &i915->gpu_error.flags);
3563
-
3564
-	return true;
3565
-}
3566
-
3567
-static void
3568
-i915_gem_retire_work_handler(struct work_struct *work)
3569
-{
3570
-	struct drm_i915_private *dev_priv =
3571
-		container_of(work, typeof(*dev_priv), gt.retire_work.work);
3572
-	struct drm_device *dev = &dev_priv->drm;
3573
-
3574
-	/* Come back later if the device is busy... */
3575
-	if (mutex_trylock(&dev->struct_mutex)) {
3576
-		i915_retire_requests(dev_priv);
3577
-		mutex_unlock(&dev->struct_mutex);
3578
-	}
3579
-
3580
-	/*
3581
-	 * Keep the retire handler running until we are finally idle.
3582
-	 * We do not need to do this test under locking as in the worst-case
3583
-	 * we queue the retire worker once too often.
3584
-	 */
3585
-	if (READ_ONCE(dev_priv->gt.awake))
3586
-		queue_delayed_work(dev_priv->wq,
3587
-				   &dev_priv->gt.retire_work,
3588
-				   round_jiffies_up_relative(HZ));
3589
-}
3590
-
3591
-static void shrink_caches(struct drm_i915_private *i915)
3592
-{
3593
-	/*
3594
-	 * kmem_cache_shrink() discards empty slabs and reorders partially
3595
-	 * filled slabs to prioritise allocating from the mostly full slabs,
3596
-	 * with the aim of reducing fragmentation.
3597
-	 */
3598
-	kmem_cache_shrink(i915->priorities);
3599
-	kmem_cache_shrink(i915->dependencies);
3600
-	kmem_cache_shrink(i915->requests);
3601
-	kmem_cache_shrink(i915->luts);
3602
-	kmem_cache_shrink(i915->vmas);
3603
-	kmem_cache_shrink(i915->objects);
3604
-}
3605
-
3606
-struct sleep_rcu_work {
3607
-	union {
3608
-		struct rcu_head rcu;
3609
-		struct work_struct work;
3610
-	};
3611
-	struct drm_i915_private *i915;
3612
-	unsigned int epoch;
3613
-};
3614
-
3615
-static inline bool
3616
-same_epoch(struct drm_i915_private *i915, unsigned int epoch)
3617
-{
3618
-	/*
3619
-	 * There is a small chance that the epoch wrapped since we started
3620
-	 * sleeping. If we assume that epoch is at least a u32, then it will
3621
-	 * take at least 2^32 * 100ms for it to wrap, or about 326 years.
3622
-	 */
3623
-	return epoch == READ_ONCE(i915->gt.epoch);
3624
-}
3625
-
3626
-static void __sleep_work(struct work_struct *work)
3627
-{
3628
-	struct sleep_rcu_work *s = container_of(work, typeof(*s), work);
3629
-	struct drm_i915_private *i915 = s->i915;
3630
-	unsigned int epoch = s->epoch;
3631
-
3632
-	kfree(s);
3633
-	if (same_epoch(i915, epoch))
3634
-		shrink_caches(i915);
3635
-}
3636
-
3637
-static void __sleep_rcu(struct rcu_head *rcu)
3638
-{
3639
-	struct sleep_rcu_work *s = container_of(rcu, typeof(*s), rcu);
3640
-	struct drm_i915_private *i915 = s->i915;
3641
-
3642
-	if (same_epoch(i915, s->epoch)) {
3643
-		INIT_WORK(&s->work, __sleep_work);
3644
-		queue_work(i915->wq, &s->work);
3645
-	} else {
3646
-		kfree(s);
3647
-	}
3648
-}
3649
-
3650
-static inline bool
3651
-new_requests_since_last_retire(const struct drm_i915_private *i915)
3652
-{
3653
-	return (READ_ONCE(i915->gt.active_requests) ||
3654
-		work_pending(&i915->gt.idle_work.work));
3655
-}
3656
-
3657
-static void assert_kernel_context_is_current(struct drm_i915_private *i915)
3658
-{
3659
-	struct intel_engine_cs *engine;
3660
-	enum intel_engine_id id;
3661
-
3662
-	if (i915_terminally_wedged(&i915->gpu_error))
3663
-		return;
3664
-
3665
-	GEM_BUG_ON(i915->gt.active_requests);
3666
-	for_each_engine(engine, i915, id) {
3667
-		GEM_BUG_ON(__i915_gem_active_peek(&engine->timeline.last_request));
3668
-		GEM_BUG_ON(engine->last_retired_context !=
3669
-			   to_intel_context(i915->kernel_context, engine));
3670
-	}
3671
-}
3672
-
3673
-static void
3674
-i915_gem_idle_work_handler(struct work_struct *work)
3675
-{
3676
-	struct drm_i915_private *dev_priv =
3677
-		container_of(work, typeof(*dev_priv), gt.idle_work.work);
3678
-	unsigned int epoch = I915_EPOCH_INVALID;
3679
-	bool rearm_hangcheck;
3680
-
3681
-	if (!READ_ONCE(dev_priv->gt.awake))
3682
-		return;
3683
-
3684
-	if (READ_ONCE(dev_priv->gt.active_requests))
3685
-		return;
3686
-
3687
-	/*
3688
-	 * Flush out the last user context, leaving only the pinned
3689
-	 * kernel context resident. When we are idling on the kernel_context,
3690
-	 * no more new requests (with a context switch) are emitted and we
3691
-	 * can finally rest. A consequence is that the idle work handler is
3692
-	 * always called at least twice before idling (and if the system is
3693
-	 * idle that implies a round trip through the retire worker).
3694
-	 */
3695
-	mutex_lock(&dev_priv->drm.struct_mutex);
3696
-	i915_gem_switch_to_kernel_context(dev_priv);
3697
-	mutex_unlock(&dev_priv->drm.struct_mutex);
3698
-
3699
-	GEM_TRACE("active_requests=%d (after switch-to-kernel-context)\n",
3700
-		  READ_ONCE(dev_priv->gt.active_requests));
3701
-
3702
-	/*
3703
-	 * Wait for last execlists context complete, but bail out in case a
3704
-	 * new request is submitted. As we don't trust the hardware, we
3705
-	 * continue on if the wait times out. This is necessary to allow
3706
-	 * the machine to suspend even if the hardware dies, and we will
3707
-	 * try to recover in resume (after depriving the hardware of power,
3708
-	 * it may be in a better mmod).
3709
-	 */
3710
-	__wait_for(if (new_requests_since_last_retire(dev_priv)) return,
3711
-		   intel_engines_are_idle(dev_priv),
3712
-		   I915_IDLE_ENGINES_TIMEOUT * 1000,
3713
-		   10, 500);
3714
-
3715
-	rearm_hangcheck =
3716
-		cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
3717
-
3718
-	if (!mutex_trylock(&dev_priv->drm.struct_mutex)) {
3719
-		/* Currently busy, come back later */
3720
-		mod_delayed_work(dev_priv->wq,
3721
-				 &dev_priv->gt.idle_work,
3722
-				 msecs_to_jiffies(50));
3723
-		goto out_rearm;
3724
-	}
3725
-
3726
-	/*
3727
-	 * New request retired after this work handler started, extend active
3728
-	 * period until next instance of the work.
3729
-	 */
3730
-	if (new_requests_since_last_retire(dev_priv))
3731
-		goto out_unlock;
3732
-
3733
-	epoch = __i915_gem_park(dev_priv);
3734
-
3735
-	assert_kernel_context_is_current(dev_priv);
3736
-
3737
-	rearm_hangcheck = false;
3738
-out_unlock:
3739
-	mutex_unlock(&dev_priv->drm.struct_mutex);
3740
-
3741
-out_rearm:
3742
-	if (rearm_hangcheck) {
3743
-		GEM_BUG_ON(!dev_priv->gt.awake);
3744
-		i915_queue_hangcheck(dev_priv);
3745
-	}
3746
-
3747
-	/*
3748
-	 * When we are idle, it is an opportune time to reap our caches.
3749
-	 * However, we have many objects that utilise RCU and the ordered
3750
-	 * i915->wq that this work is executing on. To try and flush any
3751
-	 * pending frees now we are idle, we first wait for an RCU grace
3752
-	 * period, and then queue a task (that will run last on the wq) to
3753
-	 * shrink and re-optimize the caches.
3754
-	 */
3755
-	if (same_epoch(dev_priv, epoch)) {
3756
-		struct sleep_rcu_work *s = kmalloc(sizeof(*s), GFP_KERNEL);
3757
-		if (s) {
3758
-			s->i915 = dev_priv;
3759
-			s->epoch = epoch;
3760
-			call_rcu(&s->rcu, __sleep_rcu);
3761
-		}
3762
-	}
3763
-}
3764
-
3765
-void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
3766
-{
3767
-	struct drm_i915_private *i915 = to_i915(gem->dev);
3768
-	struct drm_i915_gem_object *obj = to_intel_bo(gem);
3769
-	struct drm_i915_file_private *fpriv = file->driver_priv;
3770
-	struct i915_lut_handle *lut, *ln;
3771
-
3772
-	mutex_lock(&i915->drm.struct_mutex);
3773
-
3774
-	list_for_each_entry_safe(lut, ln, &obj->lut_list, obj_link) {
3775
-		struct i915_gem_context *ctx = lut->ctx;
3776
-		struct i915_vma *vma;
3777
-
3778
-		GEM_BUG_ON(ctx->file_priv == ERR_PTR(-EBADF));
3779
-		if (ctx->file_priv != fpriv)
3780
-			continue;
3781
-
3782
-		vma = radix_tree_delete(&ctx->handles_vma, lut->handle);
3783
-		GEM_BUG_ON(vma->obj != obj);
3784
-
3785
-		/* We allow the process to have multiple handles to the same
3786
-		 * vma, in the same fd namespace, by virtue of flink/open.
3787
-		 */
3788
-		GEM_BUG_ON(!vma->open_count);
3789
-		if (!--vma->open_count && !i915_vma_is_ggtt(vma))
3790
-			i915_vma_close(vma);
3791
-
3792
-		list_del(&lut->obj_link);
3793
-		list_del(&lut->ctx_link);
3794
-
3795
-		kmem_cache_free(i915->luts, lut);
3796
-		__i915_gem_object_release_unless_active(obj);
3797
-	}
3798
-
3799
-	mutex_unlock(&i915->drm.struct_mutex);
3800
-}
3801
-
3802
-static unsigned long to_wait_timeout(s64 timeout_ns)
3803
-{
3804
-	if (timeout_ns < 0)
3805
-		return MAX_SCHEDULE_TIMEOUT;
3806
-
3807
-	if (timeout_ns == 0)
3808
-		return 0;
3809
-
3810
-	return nsecs_to_jiffies_timeout(timeout_ns);
3811
-}
3812
-
3813
-/**
3814
- * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
3815
- * @dev: drm device pointer
3816
- * @data: ioctl data blob
3817
- * @file: drm file pointer
3818
- *
3819
- * Returns 0 if successful, else an error is returned with the remaining time in
3820
- * the timeout parameter.
3821
- *  -ETIME: object is still busy after timeout
3822
- *  -ERESTARTSYS: signal interrupted the wait
3823
- *  -ENONENT: object doesn't exist
3824
- * Also possible, but rare:
3825
- *  -EAGAIN: incomplete, restart syscall
3826
- *  -ENOMEM: damn
3827
- *  -ENODEV: Internal IRQ fail
3828
- *  -E?: The add request failed
3829
- *
3830
- * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
3831
- * non-zero timeout parameter the wait ioctl will wait for the given number of
3832
- * nanoseconds on an object becoming unbusy. Since the wait itself does so
3833
- * without holding struct_mutex the object may become re-busied before this
3834
- * function completes. A similar but shorter * race condition exists in the busy
3835
- * ioctl
3836
- */
3837
-int
3838
-i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
3839
-{
3840
-	struct drm_i915_gem_wait *args = data;
3841
-	struct drm_i915_gem_object *obj;
3842
-	ktime_t start;
3843
-	long ret;
3844
-
3845
-	if (args->flags != 0)
3846
-		return -EINVAL;
3847
-
3848
-	obj = i915_gem_object_lookup(file, args->bo_handle);
3849
-	if (!obj)
3850
-		return -ENOENT;
3851
-
3852
-	start = ktime_get();
3853
-
3854
-	ret = i915_gem_object_wait(obj,
3855
-				   I915_WAIT_INTERRUPTIBLE | I915_WAIT_ALL,
3856
-				   to_wait_timeout(args->timeout_ns),
3857
-				   to_rps_client(file));
3858
-
3859
-	if (args->timeout_ns > 0) {
3860
-		args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
3861
-		if (args->timeout_ns < 0)
3862
-			args->timeout_ns = 0;
3863
-
3864
-		/*
3865
-		 * Apparently ktime isn't accurate enough and occasionally has a
3866
-		 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
3867
-		 * things up to make the test happy. We allow up to 1 jiffy.
3868
-		 *
3869
-		 * This is a regression from the timespec->ktime conversion.
3870
-		 */
3871
-		if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns))
3872
-			args->timeout_ns = 0;
3873
-
3874
-		/* Asked to wait beyond the jiffie/scheduler precision? */
3875
-		if (ret == -ETIME && args->timeout_ns)
3876
-			ret = -EAGAIN;
3877
-	}
3878
-
3879
-	i915_gem_object_put(obj);
3880
-	return ret;
3881
-}
3882
-
3883
-static long wait_for_timeline(struct i915_timeline *tl,
3884
-			      unsigned int flags, long timeout)
3885
-{
3886
-	struct i915_request *rq;
3887
-
3888
-	rq = i915_gem_active_get_unlocked(&tl->last_request);
3889
-	if (!rq)
3890
-		return timeout;
3891
-
3892
-	/*
3893
-	 * "Race-to-idle".
3894
-	 *
3895
-	 * Switching to the kernel context is often used a synchronous
3896
-	 * step prior to idling, e.g. in suspend for flushing all
3897
-	 * current operations to memory before sleeping. These we
3898
-	 * want to complete as quickly as possible to avoid prolonged
3899
-	 * stalls, so allow the gpu to boost to maximum clocks.
3900
-	 */
3901
-	if (flags & I915_WAIT_FOR_IDLE_BOOST)
3902
-		gen6_rps_boost(rq, NULL);
3903
-
3904
-	timeout = i915_request_wait(rq, flags, timeout);
3905
-	i915_request_put(rq);
3906
-
3907
-	return timeout;
3908
-}
3909
-
3910
-static int wait_for_engines(struct drm_i915_private *i915)
3911
-{
3912
-	if (wait_for(intel_engines_are_idle(i915), I915_IDLE_ENGINES_TIMEOUT)) {
3913
-		dev_err(i915->drm.dev,
3914
-			"Failed to idle engines, declaring wedged!\n");
3915
-		GEM_TRACE_DUMP();
3916
-		i915_gem_set_wedged(i915);
3917
-		return -EIO;
3918
-	}
3919
-
3920
-	return 0;
3921
-}
3922
-
3923
-int i915_gem_wait_for_idle(struct drm_i915_private *i915,
3924
-			   unsigned int flags, long timeout)
3925
-{
3926
-	GEM_TRACE("flags=%x (%s), timeout=%ld%s\n",
3927
-		  flags, flags & I915_WAIT_LOCKED ? "locked" : "unlocked",
3928
-		  timeout, timeout == MAX_SCHEDULE_TIMEOUT ? " (forever)" : "");
3929
-
3930
-	/* If the device is asleep, we have no requests outstanding */
3931
-	if (!READ_ONCE(i915->gt.awake))
3932
-		return 0;
3933
-
3934
-	if (flags & I915_WAIT_LOCKED) {
3935
-		struct i915_timeline *tl;
3936
-		int err;
3937
-
3938
-		lockdep_assert_held(&i915->drm.struct_mutex);
3939
-
3940
-		list_for_each_entry(tl, &i915->gt.timelines, link) {
3941
-			timeout = wait_for_timeline(tl, flags, timeout);
3942
-			if (timeout < 0)
3943
-				return timeout;
3944
-		}
3945
-
3946
-		err = wait_for_engines(i915);
3947
-		if (err)
3948
-			return err;
3949
-
3950
-		i915_retire_requests(i915);
3951
-		GEM_BUG_ON(i915->gt.active_requests);
3952
-	} else {
3953
-		struct intel_engine_cs *engine;
3954
-		enum intel_engine_id id;
3955
-
3956
-		for_each_engine(engine, i915, id) {
3957
-			struct i915_timeline *tl = &engine->timeline;
3958
-
3959
-			timeout = wait_for_timeline(tl, flags, timeout);
3960
-			if (timeout < 0)
3961
-				return timeout;
3962
-		}
3963
-	}
3964
-
3965
-	return 0;
3966
-}
3967
-
3968
-static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
3969
-{
3970
-	/*
3971
-	 * We manually flush the CPU domain so that we can override and
3972
-	 * force the flush for the display, and perform it asyncrhonously.
3973
-	 */
3974
-	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
3975
-	if (obj->cache_dirty)
3976
-		i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE);
3977
-	obj->write_domain = 0;
3978
-}
3979
-
3980
-void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)
3981
-{
3982
-	if (!READ_ONCE(obj->pin_global))
3983
-		return;
3984
-
3985
-	mutex_lock(&obj->base.dev->struct_mutex);
3986
-	__i915_gem_object_flush_for_display(obj);
3987
-	mutex_unlock(&obj->base.dev->struct_mutex);
3988
-}
3989
-
3990
-/**
3991
- * Moves a single object to the WC read, and possibly write domain.
3992
- * @obj: object to act on
3993
- * @write: ask for write access or read only
3994
- *
3995
- * This function returns when the move is complete, including waiting on
3996
- * flushes to occur.
3997
- */
3998
-int
3999
-i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)
4000
-{
4001
-	int ret;
4002
-
4003
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
4004
-
4005
-	ret = i915_gem_object_wait(obj,
4006
-				   I915_WAIT_INTERRUPTIBLE |
4007
-				   I915_WAIT_LOCKED |
4008
-				   (write ? I915_WAIT_ALL : 0),
4009
-				   MAX_SCHEDULE_TIMEOUT,
4010
-				   NULL);
4011
-	if (ret)
4012
-		return ret;
4013
-
4014
-	if (obj->write_domain == I915_GEM_DOMAIN_WC)
4015
-		return 0;
4016
-
4017
-	/* Flush and acquire obj->pages so that we are coherent through
4018
-	 * direct access in memory with previous cached writes through
4019
-	 * shmemfs and that our cache domain tracking remains valid.
4020
-	 * For example, if the obj->filp was moved to swap without us
4021
-	 * being notified and releasing the pages, we would mistakenly
4022
-	 * continue to assume that the obj remained out of the CPU cached
4023
-	 * domain.
4024
-	 */
4025
-	ret = i915_gem_object_pin_pages(obj);
4026
-	if (ret)
4027
-		return ret;
4028
-
4029
-	flush_write_domain(obj, ~I915_GEM_DOMAIN_WC);
4030
-
4031
-	/* Serialise direct access to this object with the barriers for
4032
-	 * coherent writes from the GPU, by effectively invalidating the
4033
-	 * WC domain upon first access.
4034
-	 */
4035
-	if ((obj->read_domains & I915_GEM_DOMAIN_WC) == 0)
4036
-		mb();
4037
-
4038
-	/* It should now be out of any other write domains, and we can update
4039
-	 * the domain values for our changes.
4040
-	 */
4041
-	GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0);
4042
-	obj->read_domains |= I915_GEM_DOMAIN_WC;
4043
-	if (write) {
4044
-		obj->read_domains = I915_GEM_DOMAIN_WC;
4045
-		obj->write_domain = I915_GEM_DOMAIN_WC;
4046
-		obj->mm.dirty = true;
4047
-	}
4048
-
4049
-	i915_gem_object_unpin_pages(obj);
4050
-	return 0;
4051
-}
4052
-
4053
-/**
4054
- * Moves a single object to the GTT read, and possibly write domain.
4055
- * @obj: object to act on
4056
- * @write: ask for write access or read only
4057
- *
4058
- * This function returns when the move is complete, including waiting on
4059
- * flushes to occur.
4060
- */
4061
-int
4062
-i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
4063
-{
4064
-	int ret;
4065
-
4066
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
4067
-
4068
-	ret = i915_gem_object_wait(obj,
4069
-				   I915_WAIT_INTERRUPTIBLE |
4070
-				   I915_WAIT_LOCKED |
4071
-				   (write ? I915_WAIT_ALL : 0),
4072
-				   MAX_SCHEDULE_TIMEOUT,
4073
-				   NULL);
4074
-	if (ret)
4075
-		return ret;
4076
-
4077
-	if (obj->write_domain == I915_GEM_DOMAIN_GTT)
4078
-		return 0;
4079
-
4080
-	/* Flush and acquire obj->pages so that we are coherent through
4081
-	 * direct access in memory with previous cached writes through
4082
-	 * shmemfs and that our cache domain tracking remains valid.
4083
-	 * For example, if the obj->filp was moved to swap without us
4084
-	 * being notified and releasing the pages, we would mistakenly
4085
-	 * continue to assume that the obj remained out of the CPU cached
4086
-	 * domain.
4087
-	 */
4088
-	ret = i915_gem_object_pin_pages(obj);
4089
-	if (ret)
4090
-		return ret;
4091
-
4092
-	flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT);
4093
-
4094
-	/* Serialise direct access to this object with the barriers for
4095
-	 * coherent writes from the GPU, by effectively invalidating the
4096
-	 * GTT domain upon first access.
4097
-	 */
4098
-	if ((obj->read_domains & I915_GEM_DOMAIN_GTT) == 0)
4099
-		mb();
4100
-
4101
-	/* It should now be out of any other write domains, and we can update
4102
-	 * the domain values for our changes.
4103
-	 */
4104
-	GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
4105
-	obj->read_domains |= I915_GEM_DOMAIN_GTT;
4106
-	if (write) {
4107
-		obj->read_domains = I915_GEM_DOMAIN_GTT;
4108
-		obj->write_domain = I915_GEM_DOMAIN_GTT;
4109
-		obj->mm.dirty = true;
4110
-	}
4111
-
4112
-	i915_gem_object_unpin_pages(obj);
4113
-	return 0;
4114
-}
4115
-
4116
-/**
4117
- * Changes the cache-level of an object across all VMA.
4118
- * @obj: object to act on
4119
- * @cache_level: new cache level to set for the object
4120
- *
4121
- * After this function returns, the object will be in the new cache-level
4122
- * across all GTT and the contents of the backing storage will be coherent,
4123
- * with respect to the new cache-level. In order to keep the backing storage
4124
- * coherent for all users, we only allow a single cache level to be set
4125
- * globally on the object and prevent it from being changed whilst the
4126
- * hardware is reading from the object. That is if the object is currently
4127
- * on the scanout it will be set to uncached (or equivalent display
4128
- * cache coherency) and all non-MOCS GPU access will also be uncached so
4129
- * that all direct access to the scanout remains coherent.
4130
- */
4131
-int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
4132
-				    enum i915_cache_level cache_level)
4133
-{
951
+	struct i915_ggtt *ggtt = &i915->ggtt;
4134
952
 	struct i915_vma *vma;
4135
953
 	int ret;
4136
-
4137
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
4138
-
4139
-	if (obj->cache_level == cache_level)
4140
-		return 0;
4141
-
4142
-	/* Inspect the list of currently bound VMA and unbind any that would
4143
-	 * be invalid given the new cache-level. This is principally to
4144
-	 * catch the issue of the CS prefetch crossing page boundaries and
4145
-	 * reading an invalid PTE on older architectures.
4146
-	 */
4147
-restart:
4148
-	list_for_each_entry(vma, &obj->vma_list, obj_link) {
4149
-		if (!drm_mm_node_allocated(&vma->node))
4150
-			continue;
4151
-
4152
-		if (i915_vma_is_pinned(vma)) {
4153
-			DRM_DEBUG("can not change the cache level of pinned objects\n");
4154
-			return -EBUSY;
4155
-		}
4156
-
4157
-		if (!i915_vma_is_closed(vma) &&
4158
-		    i915_gem_valid_gtt_space(vma, cache_level))
4159
-			continue;
4160
-
4161
-		ret = i915_vma_unbind(vma);
4162
-		if (ret)
4163
-			return ret;
4164
-
4165
-		/* As unbinding may affect other elements in the
4166
-		 * obj->vma_list (due to side-effects from retiring
4167
-		 * an active vma), play safe and restart the iterator.
4168
-		 */
4169
-		goto restart;
4170
-	}
4171
-
4172
-	/* We can reuse the existing drm_mm nodes but need to change the
4173
-	 * cache-level on the PTE. We could simply unbind them all and
4174
-	 * rebind with the correct cache-level on next use. However since
4175
-	 * we already have a valid slot, dma mapping, pages etc, we may as
4176
-	 * rewrite the PTE in the belief that doing so tramples upon less
4177
-	 * state and so involves less work.
4178
-	 */
4179
-	if (obj->bind_count) {
4180
-		/* Before we change the PTE, the GPU must not be accessing it.
4181
-		 * If we wait upon the object, we know that all the bound
4182
-		 * VMA are no longer active.
4183
-		 */
4184
-		ret = i915_gem_object_wait(obj,
4185
-					   I915_WAIT_INTERRUPTIBLE |
4186
-					   I915_WAIT_LOCKED |
4187
-					   I915_WAIT_ALL,
4188
-					   MAX_SCHEDULE_TIMEOUT,
4189
-					   NULL);
4190
-		if (ret)
4191
-			return ret;
4192
-
4193
-		if (!HAS_LLC(to_i915(obj->base.dev)) &&
4194
-		    cache_level != I915_CACHE_NONE) {
4195
-			/* Access to snoopable pages through the GTT is
4196
-			 * incoherent and on some machines causes a hard
4197
-			 * lockup. Relinquish the CPU mmaping to force
4198
-			 * userspace to refault in the pages and we can
4199
-			 * then double check if the GTT mapping is still
4200
-			 * valid for that pointer access.
4201
-			 */
4202
-			i915_gem_release_mmap(obj);
4203
-
4204
-			/* As we no longer need a fence for GTT access,
4205
-			 * we can relinquish it now (and so prevent having
4206
-			 * to steal a fence from someone else on the next
4207
-			 * fence request). Note GPU activity would have
4208
-			 * dropped the fence as all snoopable access is
4209
-			 * supposed to be linear.
4210
-			 */
4211
-			for_each_ggtt_vma(vma, obj) {
4212
-				ret = i915_vma_put_fence(vma);
4213
-				if (ret)
4214
-					return ret;
4215
-			}
4216
-		} else {
4217
-			/* We either have incoherent backing store and
4218
-			 * so no GTT access or the architecture is fully
4219
-			 * coherent. In such cases, existing GTT mmaps
4220
-			 * ignore the cache bit in the PTE and we can
4221
-			 * rewrite it without confusing the GPU or having
4222
-			 * to force userspace to fault back in its mmaps.
4223
-			 */
4224
-		}
4225
-
4226
-		list_for_each_entry(vma, &obj->vma_list, obj_link) {
4227
-			if (!drm_mm_node_allocated(&vma->node))
4228
-				continue;
4229
-
4230
-			ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
4231
-			if (ret)
4232
-				return ret;
4233
-		}
4234
-	}
4235
-
4236
-	list_for_each_entry(vma, &obj->vma_list, obj_link)
4237
-		vma->node.color = cache_level;
4238
-	i915_gem_object_set_cache_coherency(obj, cache_level);
4239
-	obj->cache_dirty = true; /* Always invalidate stale cachelines */
4240
-
4241
-	return 0;
4242
-}
4243
-
4244
-int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
4245
-			       struct drm_file *file)
4246
-{
4247
-	struct drm_i915_gem_caching *args = data;
4248
-	struct drm_i915_gem_object *obj;
4249
-	int err = 0;
4250
-
4251
-	rcu_read_lock();
4252
-	obj = i915_gem_object_lookup_rcu(file, args->handle);
4253
-	if (!obj) {
4254
-		err = -ENOENT;
4255
-		goto out;
4256
-	}
4257
-
4258
-	switch (obj->cache_level) {
4259
-	case I915_CACHE_LLC:
4260
-	case I915_CACHE_L3_LLC:
4261
-		args->caching = I915_CACHING_CACHED;
4262
-		break;
4263
-
4264
-	case I915_CACHE_WT:
4265
-		args->caching = I915_CACHING_DISPLAY;
4266
-		break;
4267
-
4268
-	default:
4269
-		args->caching = I915_CACHING_NONE;
4270
-		break;
4271
-	}
4272
-out:
4273
-	rcu_read_unlock();
4274
-	return err;
4275
-}
4276
-
4277
-int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
4278
-			       struct drm_file *file)
4279
-{
4280
-	struct drm_i915_private *i915 = to_i915(dev);
4281
-	struct drm_i915_gem_caching *args = data;
4282
-	struct drm_i915_gem_object *obj;
4283
-	enum i915_cache_level level;
4284
-	int ret = 0;
4285
-
4286
-	switch (args->caching) {
4287
-	case I915_CACHING_NONE:
4288
-		level = I915_CACHE_NONE;
4289
-		break;
4290
-	case I915_CACHING_CACHED:
4291
-		/*
4292
-		 * Due to a HW issue on BXT A stepping, GPU stores via a
4293
-		 * snooped mapping may leave stale data in a corresponding CPU
4294
-		 * cacheline, whereas normally such cachelines would get
4295
-		 * invalidated.
4296
-		 */
4297
-		if (!HAS_LLC(i915) && !HAS_SNOOP(i915))
4298
-			return -ENODEV;
4299
-
4300
-		level = I915_CACHE_LLC;
4301
-		break;
4302
-	case I915_CACHING_DISPLAY:
4303
-		level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE;
4304
-		break;
4305
-	default:
4306
-		return -EINVAL;
4307
-	}
4308
-
4309
-	obj = i915_gem_object_lookup(file, args->handle);
4310
-	if (!obj)
4311
-		return -ENOENT;
4312
-
4313
-	/*
4314
-	 * The caching mode of proxy object is handled by its generator, and
4315
-	 * not allowed to be changed by userspace.
4316
-	 */
4317
-	if (i915_gem_object_is_proxy(obj)) {
4318
-		ret = -ENXIO;
4319
-		goto out;
4320
-	}
4321
-
4322
-	if (obj->cache_level == level)
4323
-		goto out;
4324
-
4325
-	ret = i915_gem_object_wait(obj,
4326
-				   I915_WAIT_INTERRUPTIBLE,
4327
-				   MAX_SCHEDULE_TIMEOUT,
4328
-				   to_rps_client(file));
4329
-	if (ret)
4330
-		goto out;
4331
-
4332
-	ret = i915_mutex_lock_interruptible(dev);
4333
-	if (ret)
4334
-		goto out;
4335
-
4336
-	ret = i915_gem_object_set_cache_level(obj, level);
4337
-	mutex_unlock(&dev->struct_mutex);
4338
-
4339
-out:
4340
-	i915_gem_object_put(obj);
4341
-	return ret;
4342
-}
4343
-
4344
-/*
4345
- * Prepare buffer for display plane (scanout, cursors, etc). Can be called from
4346
- * an uninterruptible phase (modesetting) and allows any flushes to be pipelined
4347
- * (for pageflips). We only flush the caches while preparing the buffer for
4348
- * display, the callers are responsible for frontbuffer flush.
4349
- */
4350
-struct i915_vma *
4351
-i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
4352
-				     u32 alignment,
4353
-				     const struct i915_ggtt_view *view,
4354
-				     unsigned int flags)
4355
-{
4356
-	struct i915_vma *vma;
4357
-	int ret;
4358
-
4359
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
4360
-
4361
-	/* Mark the global pin early so that we account for the
4362
-	 * display coherency whilst setting up the cache domains.
4363
-	 */
4364
-	obj->pin_global++;
4365
-
4366
-	/* The display engine is not coherent with the LLC cache on gen6.  As
4367
-	 * a result, we make sure that the pinning that is about to occur is
4368
-	 * done with uncached PTEs. This is lowest common denominator for all
4369
-	 * chipsets.
4370
-	 *
4371
-	 * However for gen6+, we could do better by using the GFDT bit instead
4372
-	 * of uncaching, which would allow us to flush all the LLC-cached data
4373
-	 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
4374
-	 */
4375
-	ret = i915_gem_object_set_cache_level(obj,
4376
-					      HAS_WT(to_i915(obj->base.dev)) ?
4377
-					      I915_CACHE_WT : I915_CACHE_NONE);
4378
-	if (ret) {
4379
-		vma = ERR_PTR(ret);
4380
-		goto err_unpin_global;
4381
-	}
4382
-
4383
-	/* As the user may map the buffer once pinned in the display plane
4384
-	 * (e.g. libkms for the bootup splash), we have to ensure that we
4385
-	 * always use map_and_fenceable for all scanout buffers. However,
4386
-	 * it may simply be too big to fit into mappable, in which case
4387
-	 * put it anyway and hope that userspace can cope (but always first
4388
-	 * try to preserve the existing ABI).
4389
-	 */
4390
-	vma = ERR_PTR(-ENOSPC);
4391
-	if ((flags & PIN_MAPPABLE) == 0 &&
4392
-	    (!view || view->type == I915_GGTT_VIEW_NORMAL))
4393
-		vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment,
4394
-					       flags |
4395
-					       PIN_MAPPABLE |
4396
-					       PIN_NONBLOCK);
4397
-	if (IS_ERR(vma))
4398
-		vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, flags);
4399
-	if (IS_ERR(vma))
4400
-		goto err_unpin_global;
4401
-
4402
-	vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
4403
-
4404
-	__i915_gem_object_flush_for_display(obj);
4405
-
4406
-	/* It should now be out of any other write domains, and we can update
4407
-	 * the domain values for our changes.
4408
-	 */
4409
-	obj->read_domains |= I915_GEM_DOMAIN_GTT;
4410
-
4411
-	return vma;
4412
-
4413
-err_unpin_global:
4414
-	obj->pin_global--;
4415
-	return vma;
4416
-}
4417
-
4418
-void
4419
-i915_gem_object_unpin_from_display_plane(struct i915_vma *vma)
4420
-{
4421
-	lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
4422
-
4423
-	if (WARN_ON(vma->obj->pin_global == 0))
4424
-		return;
4425
-
4426
-	if (--vma->obj->pin_global == 0)
4427
-		vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
4428
-
4429
-	/* Bump the LRU to try and avoid premature eviction whilst flipping  */
4430
-	i915_gem_object_bump_inactive_ggtt(vma->obj);
4431
-
4432
-	i915_vma_unpin(vma);
4433
-}
4434
-
4435
-/**
4436
- * Moves a single object to the CPU read, and possibly write domain.
4437
- * @obj: object to act on
4438
- * @write: requesting write or read-only access
4439
- *
4440
- * This function returns when the move is complete, including waiting on
4441
- * flushes to occur.
4442
- */
4443
-int
4444
-i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
4445
-{
4446
-	int ret;
4447
-
4448
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
4449
-
4450
-	ret = i915_gem_object_wait(obj,
4451
-				   I915_WAIT_INTERRUPTIBLE |
4452
-				   I915_WAIT_LOCKED |
4453
-				   (write ? I915_WAIT_ALL : 0),
4454
-				   MAX_SCHEDULE_TIMEOUT,
4455
-				   NULL);
4456
-	if (ret)
4457
-		return ret;
4458
-
4459
-	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
4460
-
4461
-	/* Flush the CPU cache if it's still invalid. */
4462
-	if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
4463
-		i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
4464
-		obj->read_domains |= I915_GEM_DOMAIN_CPU;
4465
-	}
4466
-
4467
-	/* It should now be out of any other write domains, and we can update
4468
-	 * the domain values for our changes.
4469
-	 */
4470
-	GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU);
4471
-
4472
-	/* If we're writing through the CPU, then the GPU read domains will
4473
-	 * need to be invalidated at next use.
4474
-	 */
4475
-	if (write)
4476
-		__start_cpu_write(obj);
4477
-
4478
-	return 0;
4479
-}
4480
-
4481
-/* Throttle our rendering by waiting until the ring has completed our requests
4482
- * emitted over 20 msec ago.
4483
- *
4484
- * Note that if we were to use the current jiffies each time around the loop,
4485
- * we wouldn't escape the function with any frames outstanding if the time to
4486
- * render a frame was over 20ms.
4487
- *
4488
- * This should get us reasonable parallelism between CPU and GPU but also
4489
- * relatively low latency when blocking on a particular request to finish.
4490
- */
4491
-static int
4492
-i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
4493
-{
4494
-	struct drm_i915_private *dev_priv = to_i915(dev);
4495
-	struct drm_i915_file_private *file_priv = file->driver_priv;
4496
-	unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
4497
-	struct i915_request *request, *target = NULL;
4498
-	long ret;
4499
-
4500
-	/* ABI: return -EIO if already wedged */
4501
-	if (i915_terminally_wedged(&dev_priv->gpu_error))
4502
-		return -EIO;
4503
-
4504
-	spin_lock(&file_priv->mm.lock);
4505
-	list_for_each_entry(request, &file_priv->mm.request_list, client_link) {
4506
-		if (time_after_eq(request->emitted_jiffies, recent_enough))
4507
-			break;
4508
-
4509
-		if (target) {
4510
-			list_del(&target->client_link);
4511
-			target->file_priv = NULL;
4512
-		}
4513
-
4514
-		target = request;
4515
-	}
4516
-	if (target)
4517
-		i915_request_get(target);
4518
-	spin_unlock(&file_priv->mm.lock);
4519
-
4520
-	if (target == NULL)
4521
-		return 0;
4522
-
4523
-	ret = i915_request_wait(target,
4524
-				I915_WAIT_INTERRUPTIBLE,
4525
-				MAX_SCHEDULE_TIMEOUT);
4526
-	i915_request_put(target);
4527
-
4528
-	return ret < 0 ? ret : 0;
4529
-}
4530
-
4531
-struct i915_vma *
4532
-i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
4533
-			 const struct i915_ggtt_view *view,
4534
-			 u64 size,
4535
-			 u64 alignment,
4536
-			 u64 flags)
4537
-{
4538
-	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
4539
-	struct i915_address_space *vm = &dev_priv->ggtt.vm;
4540
-
4541
-	return i915_gem_object_pin(obj, vm, view, size, alignment,
4542
-				   flags | PIN_GLOBAL);
4543
-}
4544
-
4545
-struct i915_vma *
4546
-i915_gem_object_pin(struct drm_i915_gem_object *obj,
4547
-		    struct i915_address_space *vm,
4548
-		    const struct i915_ggtt_view *view,
4549
-		    u64 size,
4550
-		    u64 alignment,
4551
-		    u64 flags)
4552
-{
4553
-	struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
4554
-	struct i915_vma *vma;
4555
-	int ret;
4556
-
4557
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
4558
954
 
4559
955
 	if (flags & PIN_MAPPABLE &&
4560
956
 	    (!view || view->type == I915_GGTT_VIEW_NORMAL)) {
4561
-		/* If the required space is larger than the available
957
+		/*
958
+		 * If the required space is larger than the available
4562
959
 		 * aperture, we will not able to find a slot for the
4563
960
 		 * object and unbinding the object now will be in
4564
961
 		 * vain. Worse, doing so may cause us to ping-pong
4565
962
 		 * the object in and out of the Global GTT and
4566
963
 		 * waste a lot of cycles under the mutex.
4567
964
 		 */
4568
-		if (obj->base.size > dev_priv->ggtt.mappable_end)
965
+		if (obj->base.size > ggtt->mappable_end)
4569
966
 			return ERR_PTR(-E2BIG);
4570
967
 
4571
-		/* If NONBLOCK is set the caller is optimistically
968
+		/*
969
+		 * If NONBLOCK is set the caller is optimistically
4572
970
 		 * trying to cache the full object within the mappable
4573
971
 		 * aperture, and *must* have a fallback in place for
4574
972
 		 * situations where we cannot bind the object. We
..
..
@@ -4584,12 +982,13 @@
4584
982
 		 * we could try to minimise harm to others.
4585
983
 		 */
4586
984
 		if (flags & PIN_NONBLOCK &&
4587
-		    obj->base.size > dev_priv->ggtt.mappable_end / 2)
985
+		    obj->base.size > ggtt->mappable_end / 2)
4588
986
 			return ERR_PTR(-ENOSPC);
4589
987
 	}
4590
988
 
4591
-	vma = i915_vma_instance(obj, vm, view);
4592
-	if (unlikely(IS_ERR(vma)))
989
+new_vma:
990
+	vma = i915_vma_instance(obj, &ggtt->vm, view);
991
+	if (IS_ERR(vma))
4593
992
 		return vma;
4594
993
 
4595
994
 	if (i915_vma_misplaced(vma, size, alignment, flags)) {
..
..
@@ -4598,166 +997,44 @@
4598
997
 				return ERR_PTR(-ENOSPC);
4599
998
 
4600
999
 			if (flags & PIN_MAPPABLE &&
4601
-			    vma->fence_size > dev_priv->ggtt.mappable_end / 2)
1000
+			    vma->fence_size > ggtt->mappable_end / 2)
4602
1001
 				return ERR_PTR(-ENOSPC);
4603
1002
 		}
4604
1003
 
4605
-		WARN(i915_vma_is_pinned(vma),
4606
-		     "bo is already pinned in ggtt with incorrect alignment:"
4607
-		     " offset=%08x, req.alignment=%llx,"
4608
-		     " req.map_and_fenceable=%d, vma->map_and_fenceable=%d\n",
4609
-		     i915_ggtt_offset(vma), alignment,
4610
-		     !!(flags & PIN_MAPPABLE),
4611
-		     i915_vma_is_map_and_fenceable(vma));
1004
+		if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) {
1005
+			discard_ggtt_vma(vma);
1006
+			goto new_vma;
1007
+		}
1008
+
4612
1009
 		ret = i915_vma_unbind(vma);
4613
1010
 		if (ret)
4614
1011
 			return ERR_PTR(ret);
4615
1012
 	}
4616
1013
 
4617
-	ret = i915_vma_pin(vma, size, alignment, flags);
1014
+	ret = i915_vma_pin_ww(vma, ww, size, alignment, flags | PIN_GLOBAL);
4618
1015
 	if (ret)
4619
1016
 		return ERR_PTR(ret);
4620
1017
 
4621
-	return vma;
4622
-}
4623
-
4624
-static __always_inline unsigned int __busy_read_flag(unsigned int id)
4625
-{
4626
-	/* Note that we could alias engines in the execbuf API, but
4627
-	 * that would be very unwise as it prevents userspace from
4628
-	 * fine control over engine selection. Ahem.
4629
-	 *
4630
-	 * This should be something like EXEC_MAX_ENGINE instead of
4631
-	 * I915_NUM_ENGINES.
4632
-	 */
4633
-	BUILD_BUG_ON(I915_NUM_ENGINES > 16);
4634
-	return 0x10000 << id;
4635
-}
4636
-
4637
-static __always_inline unsigned int __busy_write_id(unsigned int id)
4638
-{
4639
-	/* The uABI guarantees an active writer is also amongst the read
4640
-	 * engines. This would be true if we accessed the activity tracking
4641
-	 * under the lock, but as we perform the lookup of the object and
4642
-	 * its activity locklessly we can not guarantee that the last_write
4643
-	 * being active implies that we have set the same engine flag from
4644
-	 * last_read - hence we always set both read and write busy for
4645
-	 * last_write.
4646
-	 */
4647
-	return id | __busy_read_flag(id);
4648
-}
4649
-
4650
-static __always_inline unsigned int
4651
-__busy_set_if_active(const struct dma_fence *fence,
4652
-		     unsigned int (*flag)(unsigned int id))
4653
-{
4654
-	struct i915_request *rq;
4655
-
4656
-	/* We have to check the current hw status of the fence as the uABI
4657
-	 * guarantees forward progress. We could rely on the idle worker
4658
-	 * to eventually flush us, but to minimise latency just ask the
4659
-	 * hardware.
4660
-	 *
4661
-	 * Note we only report on the status of native fences.
4662
-	 */
4663
-	if (!dma_fence_is_i915(fence))
4664
-		return 0;
4665
-
4666
-	/* opencode to_request() in order to avoid const warnings */
4667
-	rq = container_of(fence, struct i915_request, fence);
4668
-	if (i915_request_completed(rq))
4669
-		return 0;
4670
-
4671
-	return flag(rq->engine->uabi_id);
4672
-}
4673
-
4674
-static __always_inline unsigned int
4675
-busy_check_reader(const struct dma_fence *fence)
4676
-{
4677
-	return __busy_set_if_active(fence, __busy_read_flag);
4678
-}
4679
-
4680
-static __always_inline unsigned int
4681
-busy_check_writer(const struct dma_fence *fence)
4682
-{
4683
-	if (!fence)
4684
-		return 0;
4685
-
4686
-	return __busy_set_if_active(fence, __busy_write_id);
4687
-}
4688
-
4689
-int
4690
-i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4691
-		    struct drm_file *file)
4692
-{
4693
-	struct drm_i915_gem_busy *args = data;
4694
-	struct drm_i915_gem_object *obj;
4695
-	struct reservation_object_list *list;
4696
-	unsigned int seq;
4697
-	int err;
4698
-
4699
-	err = -ENOENT;
4700
-	rcu_read_lock();
4701
-	obj = i915_gem_object_lookup_rcu(file, args->handle);
4702
-	if (!obj)
4703
-		goto out;
4704
-
4705
-	/* A discrepancy here is that we do not report the status of
4706
-	 * non-i915 fences, i.e. even though we may report the object as idle,
4707
-	 * a call to set-domain may still stall waiting for foreign rendering.
4708
-	 * This also means that wait-ioctl may report an object as busy,
4709
-	 * where busy-ioctl considers it idle.
4710
-	 *
4711
-	 * We trade the ability to warn of foreign fences to report on which
4712
-	 * i915 engines are active for the object.
4713
-	 *
4714
-	 * Alternatively, we can trade that extra information on read/write
4715
-	 * activity with
4716
-	 *	args->busy =
4717
-	 *		!reservation_object_test_signaled_rcu(obj->resv, true);
4718
-	 * to report the overall busyness. This is what the wait-ioctl does.
4719
-	 *
4720
-	 */
4721
-retry:
4722
-	seq = read_seqbegin(&obj->resv->seq);
4723
-
4724
-	/* Translate the exclusive fence to the READ *and* WRITE engine */
4725
-	args->busy = busy_check_writer(rcu_dereference(obj->resv->fence_excl));
4726
-
4727
-	/* Translate shared fences to READ set of engines */
4728
-	list = rcu_dereference(obj->resv->fence);
4729
-	if (list) {
4730
-		unsigned int shared_count = list->shared_count, i;
4731
-
4732
-		for (i = 0; i < shared_count; ++i) {
4733
-			struct dma_fence *fence =
4734
-				rcu_dereference(list->shared[i]);
4735
-
4736
-			args->busy |= busy_check_reader(fence);
4737
-		}
1018
+	if (vma->fence && !i915_gem_object_is_tiled(obj)) {
1019
+		mutex_lock(&ggtt->vm.mutex);
1020
+		i915_vma_revoke_fence(vma);
1021
+		mutex_unlock(&ggtt->vm.mutex);
4738
1022
 	}
4739
1023
 
4740
-	if (args->busy && read_seqretry(&obj->resv->seq, seq))
4741
-		goto retry;
1024
+	ret = i915_vma_wait_for_bind(vma);
1025
+	if (ret) {
1026
+		i915_vma_unpin(vma);
1027
+		return ERR_PTR(ret);
1028
+	}
4742
1029
 
4743
-	err = 0;
4744
-out:
4745
-	rcu_read_unlock();
4746
-	return err;
4747
-}
4748
-
4749
-int
4750
-i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4751
-			struct drm_file *file_priv)
4752
-{
4753
-	return i915_gem_ring_throttle(dev, file_priv);
1030
+	return vma;
4754
1031
 }
4755
1032
 
4756
1033
 int
4757
1034
 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4758
1035
 		       struct drm_file *file_priv)
4759
1036
 {
4760
-	struct drm_i915_private *dev_priv = to_i915(dev);
1037
+	struct drm_i915_private *i915 = to_i915(dev);
4761
1038
 	struct drm_i915_gem_madvise *args = data;
4762
1039
 	struct drm_i915_gem_object *obj;
4763
1040
 	int err;
..
..
@@ -4780,7 +1057,7 @@
4780
1057
 
4781
1058
 	if (i915_gem_object_has_pages(obj) &&
4782
1059
 	    i915_gem_object_is_tiled(obj) &&
4783
-	    dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
1060
+	    i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
4784
1061
 		if (obj->mm.madv == I915_MADV_WILLNEED) {
4785
1062
 			GEM_BUG_ON(!obj->mm.quirked);
4786
1063
 			__i915_gem_object_unpin_pages(obj);
..
..
@@ -4796,6 +1073,24 @@
4796
1073
 	if (obj->mm.madv != __I915_MADV_PURGED)
4797
1074
 		obj->mm.madv = args->madv;
4798
1075
 
1076
+	if (i915_gem_object_has_pages(obj)) {
1077
+		struct list_head *list;
1078
+
1079
+		if (i915_gem_object_is_shrinkable(obj)) {
1080
+			unsigned long flags;
1081
+
1082
+			spin_lock_irqsave(&i915->mm.obj_lock, flags);
1083
+
1084
+			if (obj->mm.madv != I915_MADV_WILLNEED)
1085
+				list = &i915->mm.purge_list;
1086
+			else
1087
+				list = &i915->mm.shrink_list;
1088
+			list_move_tail(&obj->mm.link, list);
1089
+
1090
+			spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
1091
+		}
1092
+	}
1093
+
4799
1094
 	/* if the object is no longer attached, discard its backing storage */
4800
1095
 	if (obj->mm.madv == I915_MADV_DONTNEED &&
4801
1096
 	    !i915_gem_object_has_pages(obj))
..
..
@@ -4809,793 +1104,6 @@
4809
1104
 	return err;
4810
1105
 }
4811
1106
 
4812
-static void
4813
-frontbuffer_retire(struct i915_gem_active *active, struct i915_request *request)
4814
-{
4815
-	struct drm_i915_gem_object *obj =
4816
-		container_of(active, typeof(*obj), frontbuffer_write);
4817
-
4818
-	intel_fb_obj_flush(obj, ORIGIN_CS);
4819
-}
4820
-
4821
-void i915_gem_object_init(struct drm_i915_gem_object *obj,
4822
-			  const struct drm_i915_gem_object_ops *ops)
4823
-{
4824
-	mutex_init(&obj->mm.lock);
4825
-
4826
-	INIT_LIST_HEAD(&obj->vma_list);
4827
-	INIT_LIST_HEAD(&obj->lut_list);
4828
-	INIT_LIST_HEAD(&obj->batch_pool_link);
4829
-
4830
-	obj->ops = ops;
4831
-
4832
-	reservation_object_init(&obj->__builtin_resv);
4833
-	obj->resv = &obj->__builtin_resv;
4834
-
4835
-	obj->frontbuffer_ggtt_origin = ORIGIN_GTT;
4836
-	init_request_active(&obj->frontbuffer_write, frontbuffer_retire);
4837
-
4838
-	obj->mm.madv = I915_MADV_WILLNEED;
4839
-	INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN);
4840
-	mutex_init(&obj->mm.get_page.lock);
4841
-
4842
-	i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size);
4843
-}
4844
-
4845
-static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
4846
-	.flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
4847
-		 I915_GEM_OBJECT_IS_SHRINKABLE,
4848
-
4849
-	.get_pages = i915_gem_object_get_pages_gtt,
4850
-	.put_pages = i915_gem_object_put_pages_gtt,
4851
-
4852
-	.pwrite = i915_gem_object_pwrite_gtt,
4853
-};
4854
-
4855
-static int i915_gem_object_create_shmem(struct drm_device *dev,
4856
-					struct drm_gem_object *obj,
4857
-					size_t size)
4858
-{
4859
-	struct drm_i915_private *i915 = to_i915(dev);
4860
-	unsigned long flags = VM_NORESERVE;
4861
-	struct file *filp;
4862
-
4863
-	drm_gem_private_object_init(dev, obj, size);
4864
-
4865
-	if (i915->mm.gemfs)
4866
-		filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size,
4867
-						 flags);
4868
-	else
4869
-		filp = shmem_file_setup("i915", size, flags);
4870
-
4871
-	if (IS_ERR(filp))
4872
-		return PTR_ERR(filp);
4873
-
4874
-	obj->filp = filp;
4875
-
4876
-	return 0;
4877
-}
4878
-
4879
-struct drm_i915_gem_object *
4880
-i915_gem_object_create(struct drm_i915_private *dev_priv, u64 size)
4881
-{
4882
-	struct drm_i915_gem_object *obj;
4883
-	struct address_space *mapping;
4884
-	unsigned int cache_level;
4885
-	gfp_t mask;
4886
-	int ret;
4887
-
4888
-	/* There is a prevalence of the assumption that we fit the object's
4889
-	 * page count inside a 32bit _signed_ variable. Let's document this and
4890
-	 * catch if we ever need to fix it. In the meantime, if you do spot
4891
-	 * such a local variable, please consider fixing!
4892
-	 */
4893
-	if (size >> PAGE_SHIFT > INT_MAX)
4894
-		return ERR_PTR(-E2BIG);
4895
-
4896
-	if (overflows_type(size, obj->base.size))
4897
-		return ERR_PTR(-E2BIG);
4898
-
4899
-	obj = i915_gem_object_alloc(dev_priv);
4900
-	if (obj == NULL)
4901
-		return ERR_PTR(-ENOMEM);
4902
-
4903
-	ret = i915_gem_object_create_shmem(&dev_priv->drm, &obj->base, size);
4904
-	if (ret)
4905
-		goto fail;
4906
-
4907
-	mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4908
-	if (IS_I965GM(dev_priv) || IS_I965G(dev_priv)) {
4909
-		/* 965gm cannot relocate objects above 4GiB. */
4910
-		mask &= ~__GFP_HIGHMEM;
4911
-		mask |= __GFP_DMA32;
4912
-	}
4913
-
4914
-	mapping = obj->base.filp->f_mapping;
4915
-	mapping_set_gfp_mask(mapping, mask);
4916
-	GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM));
4917
-
4918
-	i915_gem_object_init(obj, &i915_gem_object_ops);
4919
-
4920
-	obj->write_domain = I915_GEM_DOMAIN_CPU;
4921
-	obj->read_domains = I915_GEM_DOMAIN_CPU;
4922
-
4923
-	if (HAS_LLC(dev_priv))
4924
-		/* On some devices, we can have the GPU use the LLC (the CPU
4925
-		 * cache) for about a 10% performance improvement
4926
-		 * compared to uncached.  Graphics requests other than
4927
-		 * display scanout are coherent with the CPU in
4928
-		 * accessing this cache.  This means in this mode we
4929
-		 * don't need to clflush on the CPU side, and on the
4930
-		 * GPU side we only need to flush internal caches to
4931
-		 * get data visible to the CPU.
4932
-		 *
4933
-		 * However, we maintain the display planes as UC, and so
4934
-		 * need to rebind when first used as such.
4935
-		 */
4936
-		cache_level = I915_CACHE_LLC;
4937
-	else
4938
-		cache_level = I915_CACHE_NONE;
4939
-
4940
-	i915_gem_object_set_cache_coherency(obj, cache_level);
4941
-
4942
-	trace_i915_gem_object_create(obj);
4943
-
4944
-	return obj;
4945
-
4946
-fail:
4947
-	i915_gem_object_free(obj);
4948
-	return ERR_PTR(ret);
4949
-}
4950
-
4951
-static bool discard_backing_storage(struct drm_i915_gem_object *obj)
4952
-{
4953
-	/* If we are the last user of the backing storage (be it shmemfs
4954
-	 * pages or stolen etc), we know that the pages are going to be
4955
-	 * immediately released. In this case, we can then skip copying
4956
-	 * back the contents from the GPU.
4957
-	 */
4958
-
4959
-	if (obj->mm.madv != I915_MADV_WILLNEED)
4960
-		return false;
4961
-
4962
-	if (obj->base.filp == NULL)
4963
-		return true;
4964
-
4965
-	/* At first glance, this looks racy, but then again so would be
4966
-	 * userspace racing mmap against close. However, the first external
4967
-	 * reference to the filp can only be obtained through the
4968
-	 * i915_gem_mmap_ioctl() which safeguards us against the user
4969
-	 * acquiring such a reference whilst we are in the middle of
4970
-	 * freeing the object.
4971
-	 */
4972
-	return atomic_long_read(&obj->base.filp->f_count) == 1;
4973
-}
4974
-
4975
-static void __i915_gem_free_objects(struct drm_i915_private *i915,
4976
-				    struct llist_node *freed)
4977
-{
4978
-	struct drm_i915_gem_object *obj, *on;
4979
-
4980
-	intel_runtime_pm_get(i915);
4981
-	llist_for_each_entry_safe(obj, on, freed, freed) {
4982
-		struct i915_vma *vma, *vn;
4983
-
4984
-		trace_i915_gem_object_destroy(obj);
4985
-
4986
-		mutex_lock(&i915->drm.struct_mutex);
4987
-
4988
-		GEM_BUG_ON(i915_gem_object_is_active(obj));
4989
-		list_for_each_entry_safe(vma, vn,
4990
-					 &obj->vma_list, obj_link) {
4991
-			GEM_BUG_ON(i915_vma_is_active(vma));
4992
-			vma->flags &= ~I915_VMA_PIN_MASK;
4993
-			i915_vma_destroy(vma);
4994
-		}
4995
-		GEM_BUG_ON(!list_empty(&obj->vma_list));
4996
-		GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma_tree));
4997
-
4998
-		/* This serializes freeing with the shrinker. Since the free
4999
-		 * is delayed, first by RCU then by the workqueue, we want the
5000
-		 * shrinker to be able to free pages of unreferenced objects,
5001
-		 * or else we may oom whilst there are plenty of deferred
5002
-		 * freed objects.
5003
-		 */
5004
-		if (i915_gem_object_has_pages(obj)) {
5005
-			spin_lock(&i915->mm.obj_lock);
5006
-			list_del_init(&obj->mm.link);
5007
-			spin_unlock(&i915->mm.obj_lock);
5008
-		}
5009
-
5010
-		mutex_unlock(&i915->drm.struct_mutex);
5011
-
5012
-		GEM_BUG_ON(obj->bind_count);
5013
-		GEM_BUG_ON(obj->userfault_count);
5014
-		GEM_BUG_ON(atomic_read(&obj->frontbuffer_bits));
5015
-		GEM_BUG_ON(!list_empty(&obj->lut_list));
5016
-
5017
-		if (obj->ops->release)
5018
-			obj->ops->release(obj);
5019
-
5020
-		if (WARN_ON(i915_gem_object_has_pinned_pages(obj)))
5021
-			atomic_set(&obj->mm.pages_pin_count, 0);
5022
-		__i915_gem_object_put_pages(obj, I915_MM_NORMAL);
5023
-		GEM_BUG_ON(i915_gem_object_has_pages(obj));
5024
-
5025
-		if (obj->base.import_attach)
5026
-			drm_prime_gem_destroy(&obj->base, NULL);
5027
-
5028
-		reservation_object_fini(&obj->__builtin_resv);
5029
-		drm_gem_object_release(&obj->base);
5030
-		i915_gem_info_remove_obj(i915, obj->base.size);
5031
-
5032
-		kfree(obj->bit_17);
5033
-		i915_gem_object_free(obj);
5034
-
5035
-		GEM_BUG_ON(!atomic_read(&i915->mm.free_count));
5036
-		atomic_dec(&i915->mm.free_count);
5037
-
5038
-		if (on)
5039
-			cond_resched();
5040
-	}
5041
-	intel_runtime_pm_put(i915);
5042
-}
5043
-
5044
-static void i915_gem_flush_free_objects(struct drm_i915_private *i915)
5045
-{
5046
-	struct llist_node *freed;
5047
-
5048
-	/* Free the oldest, most stale object to keep the free_list short */
5049
-	freed = NULL;
5050
-	if (!llist_empty(&i915->mm.free_list)) { /* quick test for hotpath */
5051
-		/* Only one consumer of llist_del_first() allowed */
5052
-		spin_lock(&i915->mm.free_lock);
5053
-		freed = llist_del_first(&i915->mm.free_list);
5054
-		spin_unlock(&i915->mm.free_lock);
5055
-	}
5056
-	if (unlikely(freed)) {
5057
-		freed->next = NULL;
5058
-		__i915_gem_free_objects(i915, freed);
5059
-	}
5060
-}
5061
-
5062
-static void __i915_gem_free_work(struct work_struct *work)
5063
-{
5064
-	struct drm_i915_private *i915 =
5065
-		container_of(work, struct drm_i915_private, mm.free_work);
5066
-	struct llist_node *freed;
5067
-
5068
-	/*
5069
-	 * All file-owned VMA should have been released by this point through
5070
-	 * i915_gem_close_object(), or earlier by i915_gem_context_close().
5071
-	 * However, the object may also be bound into the global GTT (e.g.
5072
-	 * older GPUs without per-process support, or for direct access through
5073
-	 * the GTT either for the user or for scanout). Those VMA still need to
5074
-	 * unbound now.
5075
-	 */
5076
-
5077
-	spin_lock(&i915->mm.free_lock);
5078
-	while ((freed = llist_del_all(&i915->mm.free_list))) {
5079
-		spin_unlock(&i915->mm.free_lock);
5080
-
5081
-		__i915_gem_free_objects(i915, freed);
5082
-		if (need_resched())
5083
-			return;
5084
-
5085
-		spin_lock(&i915->mm.free_lock);
5086
-	}
5087
-	spin_unlock(&i915->mm.free_lock);
5088
-}
5089
-
5090
-static void __i915_gem_free_object_rcu(struct rcu_head *head)
5091
-{
5092
-	struct drm_i915_gem_object *obj =
5093
-		container_of(head, typeof(*obj), rcu);
5094
-	struct drm_i915_private *i915 = to_i915(obj->base.dev);
5095
-
5096
-	/*
5097
-	 * Since we require blocking on struct_mutex to unbind the freed
5098
-	 * object from the GPU before releasing resources back to the
5099
-	 * system, we can not do that directly from the RCU callback (which may
5100
-	 * be a softirq context), but must instead then defer that work onto a
5101
-	 * kthread. We use the RCU callback rather than move the freed object
5102
-	 * directly onto the work queue so that we can mix between using the
5103
-	 * worker and performing frees directly from subsequent allocations for
5104
-	 * crude but effective memory throttling.
5105
-	 */
5106
-	if (llist_add(&obj->freed, &i915->mm.free_list))
5107
-		queue_work(i915->wq, &i915->mm.free_work);
5108
-}
5109
-
5110
-void i915_gem_free_object(struct drm_gem_object *gem_obj)
5111
-{
5112
-	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
5113
-
5114
-	if (obj->mm.quirked)
5115
-		__i915_gem_object_unpin_pages(obj);
5116
-
5117
-	if (discard_backing_storage(obj))
5118
-		obj->mm.madv = I915_MADV_DONTNEED;
5119
-
5120
-	/*
5121
-	 * Before we free the object, make sure any pure RCU-only
5122
-	 * read-side critical sections are complete, e.g.
5123
-	 * i915_gem_busy_ioctl(). For the corresponding synchronized
5124
-	 * lookup see i915_gem_object_lookup_rcu().
5125
-	 */
5126
-	atomic_inc(&to_i915(obj->base.dev)->mm.free_count);
5127
-	call_rcu(&obj->rcu, __i915_gem_free_object_rcu);
5128
-}
5129
-
5130
-void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj)
5131
-{
5132
-	lockdep_assert_held(&obj->base.dev->struct_mutex);
5133
-
5134
-	if (!i915_gem_object_has_active_reference(obj) &&
5135
-	    i915_gem_object_is_active(obj))
5136
-		i915_gem_object_set_active_reference(obj);
5137
-	else
5138
-		i915_gem_object_put(obj);
5139
-}
5140
-
5141
-void i915_gem_sanitize(struct drm_i915_private *i915)
5142
-{
5143
-	int err;
5144
-
5145
-	GEM_TRACE("\n");
5146
-
5147
-	mutex_lock(&i915->drm.struct_mutex);
5148
-
5149
-	intel_runtime_pm_get(i915);
5150
-	intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
5151
-
5152
-	/*
5153
-	 * As we have just resumed the machine and woken the device up from
5154
-	 * deep PCI sleep (presumably D3_cold), assume the HW has been reset
5155
-	 * back to defaults, recovering from whatever wedged state we left it
5156
-	 * in and so worth trying to use the device once more.
5157
-	 */
5158
-	if (i915_terminally_wedged(&i915->gpu_error))
5159
-		i915_gem_unset_wedged(i915);
5160
-
5161
-	/*
5162
-	 * If we inherit context state from the BIOS or earlier occupants
5163
-	 * of the GPU, the GPU may be in an inconsistent state when we
5164
-	 * try to take over. The only way to remove the earlier state
5165
-	 * is by resetting. However, resetting on earlier gen is tricky as
5166
-	 * it may impact the display and we are uncertain about the stability
5167
-	 * of the reset, so this could be applied to even earlier gen.
5168
-	 */
5169
-	err = -ENODEV;
5170
-	if (INTEL_GEN(i915) >= 5 && intel_has_gpu_reset(i915))
5171
-		err = WARN_ON(intel_gpu_reset(i915, ALL_ENGINES));
5172
-	if (!err)
5173
-		intel_engines_sanitize(i915);
5174
-
5175
-	intel_uncore_forcewake_put(i915, FORCEWAKE_ALL);
5176
-	intel_runtime_pm_put(i915);
5177
-
5178
-	i915_gem_contexts_lost(i915);
5179
-	mutex_unlock(&i915->drm.struct_mutex);
5180
-}
5181
-
5182
-int i915_gem_suspend(struct drm_i915_private *i915)
5183
-{
5184
-	int ret;
5185
-
5186
-	GEM_TRACE("\n");
5187
-
5188
-	intel_runtime_pm_get(i915);
5189
-	intel_suspend_gt_powersave(i915);
5190
-
5191
-	mutex_lock(&i915->drm.struct_mutex);
5192
-
5193
-	/*
5194
-	 * We have to flush all the executing contexts to main memory so
5195
-	 * that they can saved in the hibernation image. To ensure the last
5196
-	 * context image is coherent, we have to switch away from it. That
5197
-	 * leaves the i915->kernel_context still active when
5198
-	 * we actually suspend, and its image in memory may not match the GPU
5199
-	 * state. Fortunately, the kernel_context is disposable and we do
5200
-	 * not rely on its state.
5201
-	 */
5202
-	if (!i915_terminally_wedged(&i915->gpu_error)) {
5203
-		ret = i915_gem_switch_to_kernel_context(i915);
5204
-		if (ret)
5205
-			goto err_unlock;
5206
-
5207
-		ret = i915_gem_wait_for_idle(i915,
5208
-					     I915_WAIT_INTERRUPTIBLE |
5209
-					     I915_WAIT_LOCKED |
5210
-					     I915_WAIT_FOR_IDLE_BOOST,
5211
-					     MAX_SCHEDULE_TIMEOUT);
5212
-		if (ret && ret != -EIO)
5213
-			goto err_unlock;
5214
-
5215
-		assert_kernel_context_is_current(i915);
5216
-	}
5217
-	i915_retire_requests(i915); /* ensure we flush after wedging */
5218
-
5219
-	mutex_unlock(&i915->drm.struct_mutex);
5220
-
5221
-	intel_uc_suspend(i915);
5222
-
5223
-	cancel_delayed_work_sync(&i915->gpu_error.hangcheck_work);
5224
-	cancel_delayed_work_sync(&i915->gt.retire_work);
5225
-
5226
-	/*
5227
-	 * As the idle_work is rearming if it detects a race, play safe and
5228
-	 * repeat the flush until it is definitely idle.
5229
-	 */
5230
-	drain_delayed_work(&i915->gt.idle_work);
5231
-
5232
-	/*
5233
-	 * Assert that we successfully flushed all the work and
5234
-	 * reset the GPU back to its idle, low power state.
5235
-	 */
5236
-	WARN_ON(i915->gt.awake);
5237
-	if (WARN_ON(!intel_engines_are_idle(i915)))
5238
-		i915_gem_set_wedged(i915); /* no hope, discard everything */
5239
-
5240
-	intel_runtime_pm_put(i915);
5241
-	return 0;
5242
-
5243
-err_unlock:
5244
-	mutex_unlock(&i915->drm.struct_mutex);
5245
-	intel_runtime_pm_put(i915);
5246
-	return ret;
5247
-}
5248
-
5249
-void i915_gem_suspend_late(struct drm_i915_private *i915)
5250
-{
5251
-	struct drm_i915_gem_object *obj;
5252
-	struct list_head *phases[] = {
5253
-		&i915->mm.unbound_list,
5254
-		&i915->mm.bound_list,
5255
-		NULL
5256
-	}, **phase;
5257
-
5258
-	/*
5259
-	 * Neither the BIOS, ourselves or any other kernel
5260
-	 * expects the system to be in execlists mode on startup,
5261
-	 * so we need to reset the GPU back to legacy mode. And the only
5262
-	 * known way to disable logical contexts is through a GPU reset.
5263
-	 *
5264
-	 * So in order to leave the system in a known default configuration,
5265
-	 * always reset the GPU upon unload and suspend. Afterwards we then
5266
-	 * clean up the GEM state tracking, flushing off the requests and
5267
-	 * leaving the system in a known idle state.
5268
-	 *
5269
-	 * Note that is of the upmost importance that the GPU is idle and
5270
-	 * all stray writes are flushed *before* we dismantle the backing
5271
-	 * storage for the pinned objects.
5272
-	 *
5273
-	 * However, since we are uncertain that resetting the GPU on older
5274
-	 * machines is a good idea, we don't - just in case it leaves the
5275
-	 * machine in an unusable condition.
5276
-	 */
5277
-
5278
-	mutex_lock(&i915->drm.struct_mutex);
5279
-	for (phase = phases; *phase; phase++) {
5280
-		list_for_each_entry(obj, *phase, mm.link)
5281
-			WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false));
5282
-	}
5283
-	mutex_unlock(&i915->drm.struct_mutex);
5284
-
5285
-	intel_uc_sanitize(i915);
5286
-	i915_gem_sanitize(i915);
5287
-}
5288
-
5289
-void i915_gem_resume(struct drm_i915_private *i915)
5290
-{
5291
-	GEM_TRACE("\n");
5292
-
5293
-	WARN_ON(i915->gt.awake);
5294
-
5295
-	mutex_lock(&i915->drm.struct_mutex);
5296
-	intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
5297
-
5298
-	i915_gem_restore_gtt_mappings(i915);
5299
-	i915_gem_restore_fences(i915);
5300
-
5301
-	/*
5302
-	 * As we didn't flush the kernel context before suspend, we cannot
5303
-	 * guarantee that the context image is complete. So let's just reset
5304
-	 * it and start again.
5305
-	 */
5306
-	i915->gt.resume(i915);
5307
-
5308
-	if (i915_gem_init_hw(i915))
5309
-		goto err_wedged;
5310
-
5311
-	intel_uc_resume(i915);
5312
-
5313
-	/* Always reload a context for powersaving. */
5314
-	if (i915_gem_switch_to_kernel_context(i915))
5315
-		goto err_wedged;
5316
-
5317
-out_unlock:
5318
-	intel_uncore_forcewake_put(i915, FORCEWAKE_ALL);
5319
-	mutex_unlock(&i915->drm.struct_mutex);
5320
-	return;
5321
-
5322
-err_wedged:
5323
-	if (!i915_terminally_wedged(&i915->gpu_error)) {
5324
-		DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n");
5325
-		i915_gem_set_wedged(i915);
5326
-	}
5327
-	goto out_unlock;
5328
-}
5329
-
5330
-void i915_gem_init_swizzling(struct drm_i915_private *dev_priv)
5331
-{
5332
-	if (INTEL_GEN(dev_priv) < 5 ||
5333
-	    dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
5334
-		return;
5335
-
5336
-	I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
5337
-				 DISP_TILE_SURFACE_SWIZZLING);
5338
-
5339
-	if (IS_GEN5(dev_priv))
5340
-		return;
5341
-
5342
-	I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
5343
-	if (IS_GEN6(dev_priv))
5344
-		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
5345
-	else if (IS_GEN7(dev_priv))
5346
-		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
5347
-	else if (IS_GEN8(dev_priv))
5348
-		I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
5349
-	else
5350
-		BUG();
5351
-}
5352
-
5353
-static void init_unused_ring(struct drm_i915_private *dev_priv, u32 base)
5354
-{
5355
-	I915_WRITE(RING_CTL(base), 0);
5356
-	I915_WRITE(RING_HEAD(base), 0);
5357
-	I915_WRITE(RING_TAIL(base), 0);
5358
-	I915_WRITE(RING_START(base), 0);
5359
-}
5360
-
5361
-static void init_unused_rings(struct drm_i915_private *dev_priv)
5362
-{
5363
-	if (IS_I830(dev_priv)) {
5364
-		init_unused_ring(dev_priv, PRB1_BASE);
5365
-		init_unused_ring(dev_priv, SRB0_BASE);
5366
-		init_unused_ring(dev_priv, SRB1_BASE);
5367
-		init_unused_ring(dev_priv, SRB2_BASE);
5368
-		init_unused_ring(dev_priv, SRB3_BASE);
5369
-	} else if (IS_GEN2(dev_priv)) {
5370
-		init_unused_ring(dev_priv, SRB0_BASE);
5371
-		init_unused_ring(dev_priv, SRB1_BASE);
5372
-	} else if (IS_GEN3(dev_priv)) {
5373
-		init_unused_ring(dev_priv, PRB1_BASE);
5374
-		init_unused_ring(dev_priv, PRB2_BASE);
5375
-	}
5376
-}
5377
-
5378
-static int __i915_gem_restart_engines(void *data)
5379
-{
5380
-	struct drm_i915_private *i915 = data;
5381
-	struct intel_engine_cs *engine;
5382
-	enum intel_engine_id id;
5383
-	int err;
5384
-
5385
-	for_each_engine(engine, i915, id) {
5386
-		err = engine->init_hw(engine);
5387
-		if (err) {
5388
-			DRM_ERROR("Failed to restart %s (%d)\n",
5389
-				  engine->name, err);
5390
-			return err;
5391
-		}
5392
-	}
5393
-
5394
-	return 0;
5395
-}
5396
-
5397
-int i915_gem_init_hw(struct drm_i915_private *dev_priv)
5398
-{
5399
-	int ret;
5400
-
5401
-	dev_priv->gt.last_init_time = ktime_get();
5402
-
5403
-	/* Double layer security blanket, see i915_gem_init() */
5404
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5405
-
5406
-	if (HAS_EDRAM(dev_priv) && INTEL_GEN(dev_priv) < 9)
5407
-		I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
5408
-
5409
-	if (IS_HASWELL(dev_priv))
5410
-		I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev_priv) ?
5411
-			   LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
5412
-
5413
-	if (HAS_PCH_NOP(dev_priv)) {
5414
-		if (IS_IVYBRIDGE(dev_priv)) {
5415
-			u32 temp = I915_READ(GEN7_MSG_CTL);
5416
-			temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
5417
-			I915_WRITE(GEN7_MSG_CTL, temp);
5418
-		} else if (INTEL_GEN(dev_priv) >= 7) {
5419
-			u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
5420
-			temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
5421
-			I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
5422
-		}
5423
-	}
5424
-
5425
-	intel_gt_workarounds_apply(dev_priv);
5426
-
5427
-	i915_gem_init_swizzling(dev_priv);
5428
-
5429
-	/*
5430
-	 * At least 830 can leave some of the unused rings
5431
-	 * "active" (ie. head != tail) after resume which
5432
-	 * will prevent c3 entry. Makes sure all unused rings
5433
-	 * are totally idle.
5434
-	 */
5435
-	init_unused_rings(dev_priv);
5436
-
5437
-	BUG_ON(!dev_priv->kernel_context);
5438
-	if (i915_terminally_wedged(&dev_priv->gpu_error)) {
5439
-		ret = -EIO;
5440
-		goto out;
5441
-	}
5442
-
5443
-	ret = i915_ppgtt_init_hw(dev_priv);
5444
-	if (ret) {
5445
-		DRM_ERROR("Enabling PPGTT failed (%d)\n", ret);
5446
-		goto out;
5447
-	}
5448
-
5449
-	ret = intel_wopcm_init_hw(&dev_priv->wopcm);
5450
-	if (ret) {
5451
-		DRM_ERROR("Enabling WOPCM failed (%d)\n", ret);
5452
-		goto out;
5453
-	}
5454
-
5455
-	/* We can't enable contexts until all firmware is loaded */
5456
-	ret = intel_uc_init_hw(dev_priv);
5457
-	if (ret) {
5458
-		DRM_ERROR("Enabling uc failed (%d)\n", ret);
5459
-		goto out;
5460
-	}
5461
-
5462
-	intel_mocs_init_l3cc_table(dev_priv);
5463
-
5464
-	/* Only when the HW is re-initialised, can we replay the requests */
5465
-	ret = __i915_gem_restart_engines(dev_priv);
5466
-	if (ret)
5467
-		goto cleanup_uc;
5468
-
5469
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5470
-
5471
-	return 0;
5472
-
5473
-cleanup_uc:
5474
-	intel_uc_fini_hw(dev_priv);
5475
-out:
5476
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5477
-
5478
-	return ret;
5479
-}
5480
-
5481
-static int __intel_engines_record_defaults(struct drm_i915_private *i915)
5482
-{
5483
-	struct i915_gem_context *ctx;
5484
-	struct intel_engine_cs *engine;
5485
-	enum intel_engine_id id;
5486
-	int err;
5487
-
5488
-	/*
5489
-	 * As we reset the gpu during very early sanitisation, the current
5490
-	 * register state on the GPU should reflect its defaults values.
5491
-	 * We load a context onto the hw (with restore-inhibit), then switch
5492
-	 * over to a second context to save that default register state. We
5493
-	 * can then prime every new context with that state so they all start
5494
-	 * from the same default HW values.
5495
-	 */
5496
-
5497
-	ctx = i915_gem_context_create_kernel(i915, 0);
5498
-	if (IS_ERR(ctx))
5499
-		return PTR_ERR(ctx);
5500
-
5501
-	for_each_engine(engine, i915, id) {
5502
-		struct i915_request *rq;
5503
-
5504
-		rq = i915_request_alloc(engine, ctx);
5505
-		if (IS_ERR(rq)) {
5506
-			err = PTR_ERR(rq);
5507
-			goto out_ctx;
5508
-		}
5509
-
5510
-		err = 0;
5511
-		if (engine->init_context)
5512
-			err = engine->init_context(rq);
5513
-
5514
-		i915_request_add(rq);
5515
-		if (err)
5516
-			goto err_active;
5517
-	}
5518
-
5519
-	err = i915_gem_switch_to_kernel_context(i915);
5520
-	if (err)
5521
-		goto err_active;
5522
-
5523
-	if (i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED, HZ / 5)) {
5524
-		i915_gem_set_wedged(i915);
5525
-		err = -EIO; /* Caller will declare us wedged */
5526
-		goto err_active;
5527
-	}
5528
-
5529
-	assert_kernel_context_is_current(i915);
5530
-
5531
-	for_each_engine(engine, i915, id) {
5532
-		struct i915_vma *state;
5533
-
5534
-		state = to_intel_context(ctx, engine)->state;
5535
-		if (!state)
5536
-			continue;
5537
-
5538
-		/*
5539
-		 * As we will hold a reference to the logical state, it will
5540
-		 * not be torn down with the context, and importantly the
5541
-		 * object will hold onto its vma (making it possible for a
5542
-		 * stray GTT write to corrupt our defaults). Unmap the vma
5543
-		 * from the GTT to prevent such accidents and reclaim the
5544
-		 * space.
5545
-		 */
5546
-		err = i915_vma_unbind(state);
5547
-		if (err)
5548
-			goto err_active;
5549
-
5550
-		err = i915_gem_object_set_to_cpu_domain(state->obj, false);
5551
-		if (err)
5552
-			goto err_active;
5553
-
5554
-		engine->default_state = i915_gem_object_get(state->obj);
5555
-	}
5556
-
5557
-	if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) {
5558
-		unsigned int found = intel_engines_has_context_isolation(i915);
5559
-
5560
-		/*
5561
-		 * Make sure that classes with multiple engine instances all
5562
-		 * share the same basic configuration.
5563
-		 */
5564
-		for_each_engine(engine, i915, id) {
5565
-			unsigned int bit = BIT(engine->uabi_class);
5566
-			unsigned int expected = engine->default_state ? bit : 0;
5567
-
5568
-			if ((found & bit) != expected) {
5569
-				DRM_ERROR("mismatching default context state for class %d on engine %s\n",
5570
-					  engine->uabi_class, engine->name);
5571
-			}
5572
-		}
5573
-	}
5574
-
5575
-out_ctx:
5576
-	i915_gem_context_set_closed(ctx);
5577
-	i915_gem_context_put(ctx);
5578
-	return err;
5579
-
5580
-err_active:
5581
-	/*
5582
-	 * If we have to abandon now, we expect the engines to be idle
5583
-	 * and ready to be torn-down. First try to flush any remaining
5584
-	 * request, ensure we are pointing at the kernel context and
5585
-	 * then remove it.
5586
-	 */
5587
-	if (WARN_ON(i915_gem_switch_to_kernel_context(i915)))
5588
-		goto out_ctx;
5589
-
5590
-	if (WARN_ON(i915_gem_wait_for_idle(i915,
5591
-					   I915_WAIT_LOCKED,
5592
-					   MAX_SCHEDULE_TIMEOUT)))
5593
-		goto out_ctx;
5594
-
5595
-	i915_gem_contexts_lost(i915);
5596
-	goto out_ctx;
5597
-}
5598
-
5599
1107
 int i915_gem_init(struct drm_i915_private *dev_priv)
5600
1108
 {
5601
1109
 	int ret;
..
..
@@ -5605,64 +1113,18 @@
5605
1113
 		mkwrite_device_info(dev_priv)->page_sizes =
5606
1114
 			I915_GTT_PAGE_SIZE_4K;
5607
1115
 
5608
-	dev_priv->mm.unordered_timeline = dma_fence_context_alloc(1);
5609
-
5610
-	if (HAS_LOGICAL_RING_CONTEXTS(dev_priv)) {
5611
-		dev_priv->gt.resume = intel_lr_context_resume;
5612
-		dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup;
5613
-	} else {
5614
-		dev_priv->gt.resume = intel_legacy_submission_resume;
5615
-		dev_priv->gt.cleanup_engine = intel_engine_cleanup;
5616
-	}
5617
-
5618
1116
 	ret = i915_gem_init_userptr(dev_priv);
5619
1117
 	if (ret)
5620
1118
 		return ret;
5621
1119
 
5622
-	ret = intel_uc_init_misc(dev_priv);
5623
-	if (ret)
5624
-		return ret;
1120
+	intel_uc_fetch_firmwares(&dev_priv->gt.uc);
1121
+	intel_wopcm_init(&dev_priv->wopcm);
5625
1122
 
5626
-	ret = intel_wopcm_init(&dev_priv->wopcm);
5627
-	if (ret)
5628
-		goto err_uc_misc;
5629
-
5630
-	/* This is just a security blanket to placate dragons.
5631
-	 * On some systems, we very sporadically observe that the first TLBs
5632
-	 * used by the CS may be stale, despite us poking the TLB reset. If
5633
-	 * we hold the forcewake during initialisation these problems
5634
-	 * just magically go away.
5635
-	 */
5636
-	mutex_lock(&dev_priv->drm.struct_mutex);
5637
-	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5638
-
5639
-	ret = i915_gem_init_ggtt(dev_priv);
1123
+	ret = i915_init_ggtt(dev_priv);
5640
1124
 	if (ret) {
5641
1125
 		GEM_BUG_ON(ret == -EIO);
5642
1126
 		goto err_unlock;
5643
1127
 	}
5644
-
5645
-	ret = i915_gem_contexts_init(dev_priv);
5646
-	if (ret) {
5647
-		GEM_BUG_ON(ret == -EIO);
5648
-		goto err_ggtt;
5649
-	}
5650
-
5651
-	ret = intel_engines_init(dev_priv);
5652
-	if (ret) {
5653
-		GEM_BUG_ON(ret == -EIO);
5654
-		goto err_context;
5655
-	}
5656
-
5657
-	intel_init_gt_powersave(dev_priv);
5658
-
5659
-	ret = intel_uc_init(dev_priv);
5660
-	if (ret)
5661
-		goto err_pm;
5662
-
5663
-	ret = i915_gem_init_hw(dev_priv);
5664
-	if (ret)
5665
-		goto err_uc_init;
5666
1128
 
5667
1129
 	/*
5668
1130
 	 * Despite its name intel_init_clock_gating applies both display
..
..
@@ -5675,22 +1137,9 @@
5675
1137
 	 */
5676
1138
 	intel_init_clock_gating(dev_priv);
5677
1139
 
5678
-	ret = __intel_engines_record_defaults(dev_priv);
1140
+	ret = intel_gt_init(&dev_priv->gt);
5679
1141
 	if (ret)
5680
-		goto err_init_hw;
5681
-
5682
-	if (i915_inject_load_failure()) {
5683
-		ret = -ENODEV;
5684
-		goto err_init_hw;
5685
-	}
5686
-
5687
-	if (i915_inject_load_failure()) {
5688
-		ret = -EIO;
5689
-		goto err_init_hw;
5690
-	}
5691
-
5692
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5693
-	mutex_unlock(&dev_priv->drm.struct_mutex);
1142
+		goto err_unlock;
5694
1143
 
5695
1144
 	return 0;
5696
1145
 
..
..
@@ -5700,222 +1149,96 @@
5700
1149
 	 * HW as irrevisibly wedged, but keep enough state around that the
5701
1150
 	 * driver doesn't explode during runtime.
5702
1151
 	 */
5703
-err_init_hw:
5704
-	mutex_unlock(&dev_priv->drm.struct_mutex);
5705
-
5706
-	WARN_ON(i915_gem_suspend(dev_priv));
5707
-	i915_gem_suspend_late(dev_priv);
5708
-
1152
+err_unlock:
5709
1153
 	i915_gem_drain_workqueue(dev_priv);
5710
1154
 
5711
-	mutex_lock(&dev_priv->drm.struct_mutex);
5712
-	intel_uc_fini_hw(dev_priv);
5713
-err_uc_init:
5714
-	intel_uc_fini(dev_priv);
5715
-err_pm:
5716
1155
 	if (ret != -EIO) {
5717
-		intel_cleanup_gt_powersave(dev_priv);
5718
-		i915_gem_cleanup_engines(dev_priv);
5719
-	}
5720
-err_context:
5721
-	if (ret != -EIO)
5722
-		i915_gem_contexts_fini(dev_priv);
5723
-err_ggtt:
5724
-err_unlock:
5725
-	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5726
-	mutex_unlock(&dev_priv->drm.struct_mutex);
5727
-
5728
-err_uc_misc:
5729
-	intel_uc_fini_misc(dev_priv);
5730
-
5731
-	if (ret != -EIO)
1156
+		intel_uc_cleanup_firmwares(&dev_priv->gt.uc);
5732
1157
 		i915_gem_cleanup_userptr(dev_priv);
1158
+	}
5733
1159
 
5734
1160
 	if (ret == -EIO) {
5735
-		mutex_lock(&dev_priv->drm.struct_mutex);
5736
-
5737
1161
 		/*
5738
-		 * Allow engine initialisation to fail by marking the GPU as
5739
-		 * wedged. But we only want to do this where the GPU is angry,
1162
+		 * Allow engines or uC initialisation to fail by marking the GPU
1163
+		 * as wedged. But we only want to do this when the GPU is angry,
5740
1164
 		 * for all other failure, such as an allocation failure, bail.
5741
1165
 		 */
5742
-		if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
5743
-			i915_load_error(dev_priv,
5744
-					"Failed to initialize GPU, declaring it wedged!\n");
5745
-			i915_gem_set_wedged(dev_priv);
1166
+		if (!intel_gt_is_wedged(&dev_priv->gt)) {
1167
+			i915_probe_error(dev_priv,
1168
+					 "Failed to initialize GPU, declaring it wedged!\n");
1169
+			intel_gt_set_wedged(&dev_priv->gt);
5746
1170
 		}
5747
1171
 
5748
1172
 		/* Minimal basic recovery for KMS */
5749
1173
 		ret = i915_ggtt_enable_hw(dev_priv);
5750
-		i915_gem_restore_gtt_mappings(dev_priv);
5751
-		i915_gem_restore_fences(dev_priv);
1174
+		i915_ggtt_resume(&dev_priv->ggtt);
5752
1175
 		intel_init_clock_gating(dev_priv);
5753
-
5754
-		mutex_unlock(&dev_priv->drm.struct_mutex);
5755
1176
 	}
5756
1177
 
5757
1178
 	i915_gem_drain_freed_objects(dev_priv);
5758
1179
 	return ret;
5759
1180
 }
5760
1181
 
5761
-void i915_gem_fini(struct drm_i915_private *dev_priv)
1182
+void i915_gem_driver_register(struct drm_i915_private *i915)
5762
1183
 {
1184
+	i915_gem_driver_register__shrinker(i915);
1185
+
1186
+	intel_engines_driver_register(i915);
1187
+}
1188
+
1189
+void i915_gem_driver_unregister(struct drm_i915_private *i915)
1190
+{
1191
+	i915_gem_driver_unregister__shrinker(i915);
1192
+}
1193
+
1194
+void i915_gem_driver_remove(struct drm_i915_private *dev_priv)
1195
+{
1196
+	intel_wakeref_auto_fini(&dev_priv->ggtt.userfault_wakeref);
1197
+
5763
1198
 	i915_gem_suspend_late(dev_priv);
5764
-	intel_disable_gt_powersave(dev_priv);
1199
+	intel_gt_driver_remove(&dev_priv->gt);
1200
+	dev_priv->uabi_engines = RB_ROOT;
5765
1201
 
5766
1202
 	/* Flush any outstanding unpin_work. */
5767
1203
 	i915_gem_drain_workqueue(dev_priv);
5768
1204
 
5769
-	mutex_lock(&dev_priv->drm.struct_mutex);
5770
-	intel_uc_fini_hw(dev_priv);
5771
-	intel_uc_fini(dev_priv);
5772
-	i915_gem_cleanup_engines(dev_priv);
5773
-	i915_gem_contexts_fini(dev_priv);
5774
-	mutex_unlock(&dev_priv->drm.struct_mutex);
1205
+	i915_gem_drain_freed_objects(dev_priv);
1206
+}
5775
1207
 
5776
-	intel_cleanup_gt_powersave(dev_priv);
1208
+void i915_gem_driver_release(struct drm_i915_private *dev_priv)
1209
+{
1210
+	i915_gem_driver_release__contexts(dev_priv);
5777
1211
 
5778
-	intel_uc_fini_misc(dev_priv);
1212
+	intel_gt_driver_release(&dev_priv->gt);
1213
+
1214
+	intel_wa_list_free(&dev_priv->gt_wa_list);
1215
+
1216
+	intel_uc_cleanup_firmwares(&dev_priv->gt.uc);
5779
1217
 	i915_gem_cleanup_userptr(dev_priv);
5780
1218
 
5781
1219
 	i915_gem_drain_freed_objects(dev_priv);
5782
1220
 
5783
-	WARN_ON(!list_empty(&dev_priv->contexts.list));
5784
-}
5785
-
5786
-void i915_gem_init_mmio(struct drm_i915_private *i915)
5787
-{
5788
-	i915_gem_sanitize(i915);
5789
-}
5790
-
5791
-void
5792
-i915_gem_cleanup_engines(struct drm_i915_private *dev_priv)
5793
-{
5794
-	struct intel_engine_cs *engine;
5795
-	enum intel_engine_id id;
5796
-
5797
-	for_each_engine(engine, dev_priv, id)
5798
-		dev_priv->gt.cleanup_engine(engine);
5799
-}
5800
-
5801
-void
5802
-i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
5803
-{
5804
-	int i;
5805
-
5806
-	if (INTEL_GEN(dev_priv) >= 7 && !IS_VALLEYVIEW(dev_priv) &&
5807
-	    !IS_CHERRYVIEW(dev_priv))
5808
-		dev_priv->num_fence_regs = 32;
5809
-	else if (INTEL_GEN(dev_priv) >= 4 ||
5810
-		 IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
5811
-		 IS_G33(dev_priv) || IS_PINEVIEW(dev_priv))
5812
-		dev_priv->num_fence_regs = 16;
5813
-	else
5814
-		dev_priv->num_fence_regs = 8;
5815
-
5816
-	if (intel_vgpu_active(dev_priv))
5817
-		dev_priv->num_fence_regs =
5818
-				I915_READ(vgtif_reg(avail_rs.fence_num));
5819
-
5820
-	/* Initialize fence registers to zero */
5821
-	for (i = 0; i < dev_priv->num_fence_regs; i++) {
5822
-		struct drm_i915_fence_reg *fence = &dev_priv->fence_regs[i];
5823
-
5824
-		fence->i915 = dev_priv;
5825
-		fence->id = i;
5826
-		list_add_tail(&fence->link, &dev_priv->mm.fence_list);
5827
-	}
5828
-	i915_gem_restore_fences(dev_priv);
5829
-
5830
-	i915_gem_detect_bit_6_swizzle(dev_priv);
1221
+	drm_WARN_ON(&dev_priv->drm, !list_empty(&dev_priv->gem.contexts.list));
5831
1222
 }
5832
1223
 
5833
1224
 static void i915_gem_init__mm(struct drm_i915_private *i915)
5834
1225
 {
5835
-	spin_lock_init(&i915->mm.object_stat_lock);
5836
1226
 	spin_lock_init(&i915->mm.obj_lock);
5837
-	spin_lock_init(&i915->mm.free_lock);
5838
1227
 
5839
1228
 	init_llist_head(&i915->mm.free_list);
5840
1229
 
5841
-	INIT_LIST_HEAD(&i915->mm.unbound_list);
5842
-	INIT_LIST_HEAD(&i915->mm.bound_list);
5843
-	INIT_LIST_HEAD(&i915->mm.fence_list);
5844
-	INIT_LIST_HEAD(&i915->mm.userfault_list);
1230
+	INIT_LIST_HEAD(&i915->mm.purge_list);
1231
+	INIT_LIST_HEAD(&i915->mm.shrink_list);
5845
1232
 
5846
-	INIT_WORK(&i915->mm.free_work, __i915_gem_free_work);
1233
+	i915_gem_init__objects(i915);
5847
1234
 }
5848
1235
 
5849
-int i915_gem_init_early(struct drm_i915_private *dev_priv)
1236
+void i915_gem_init_early(struct drm_i915_private *dev_priv)
5850
1237
 {
5851
-	int err = -ENOMEM;
5852
-
5853
-	dev_priv->objects = KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN);
5854
-	if (!dev_priv->objects)
5855
-		goto err_out;
5856
-
5857
-	dev_priv->vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN);
5858
-	if (!dev_priv->vmas)
5859
-		goto err_objects;
5860
-
5861
-	dev_priv->luts = KMEM_CACHE(i915_lut_handle, 0);
5862
-	if (!dev_priv->luts)
5863
-		goto err_vmas;
5864
-
5865
-	dev_priv->requests = KMEM_CACHE(i915_request,
5866
-					SLAB_HWCACHE_ALIGN |
5867
-					SLAB_RECLAIM_ACCOUNT |
5868
-					SLAB_TYPESAFE_BY_RCU);
5869
-	if (!dev_priv->requests)
5870
-		goto err_luts;
5871
-
5872
-	dev_priv->dependencies = KMEM_CACHE(i915_dependency,
5873
-					    SLAB_HWCACHE_ALIGN |
5874
-					    SLAB_RECLAIM_ACCOUNT);
5875
-	if (!dev_priv->dependencies)
5876
-		goto err_requests;
5877
-
5878
-	dev_priv->priorities = KMEM_CACHE(i915_priolist, SLAB_HWCACHE_ALIGN);
5879
-	if (!dev_priv->priorities)
5880
-		goto err_dependencies;
5881
-
5882
-	INIT_LIST_HEAD(&dev_priv->gt.timelines);
5883
-	INIT_LIST_HEAD(&dev_priv->gt.active_rings);
5884
-	INIT_LIST_HEAD(&dev_priv->gt.closed_vma);
5885
-
5886
1238
 	i915_gem_init__mm(dev_priv);
5887
-
5888
-	INIT_DELAYED_WORK(&dev_priv->gt.retire_work,
5889
-			  i915_gem_retire_work_handler);
5890
-	INIT_DELAYED_WORK(&dev_priv->gt.idle_work,
5891
-			  i915_gem_idle_work_handler);
5892
-	init_waitqueue_head(&dev_priv->gpu_error.wait_queue);
5893
-	init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
5894
-
5895
-	atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0);
1239
+	i915_gem_init__contexts(dev_priv);
5896
1240
 
5897
1241
 	spin_lock_init(&dev_priv->fb_tracking.lock);
5898
-
5899
-	mutex_init(&dev_priv->tlb_invalidate_lock);
5900
-
5901
-	err = i915_gemfs_init(dev_priv);
5902
-	if (err)
5903
-		DRM_NOTE("Unable to create a private tmpfs mount, hugepage support will be disabled(%d).\n", err);
5904
-
5905
-	return 0;
5906
-
5907
-err_dependencies:
5908
-	kmem_cache_destroy(dev_priv->dependencies);
5909
-err_requests:
5910
-	kmem_cache_destroy(dev_priv->requests);
5911
-err_luts:
5912
-	kmem_cache_destroy(dev_priv->luts);
5913
-err_vmas:
5914
-	kmem_cache_destroy(dev_priv->vmas);
5915
-err_objects:
5916
-	kmem_cache_destroy(dev_priv->objects);
5917
-err_out:
5918
-	return err;
5919
1242
 }
5920
1243
 
5921
1244
 void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
..
..
@@ -5923,20 +1246,7 @@
5923
1246
 	i915_gem_drain_freed_objects(dev_priv);
5924
1247
 	GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list));
5925
1248
 	GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count));
5926
-	WARN_ON(dev_priv->mm.object_count);
5927
-	WARN_ON(!list_empty(&dev_priv->gt.timelines));
5928
-
5929
-	kmem_cache_destroy(dev_priv->priorities);
5930
-	kmem_cache_destroy(dev_priv->dependencies);
5931
-	kmem_cache_destroy(dev_priv->requests);
5932
-	kmem_cache_destroy(dev_priv->luts);
5933
-	kmem_cache_destroy(dev_priv->vmas);
5934
-	kmem_cache_destroy(dev_priv->objects);
5935
-
5936
-	/* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
5937
-	rcu_barrier();
5938
-
5939
-	i915_gemfs_fini(dev_priv);
1249
+	drm_WARN_ON(&dev_priv->drm, dev_priv->mm.shrink_count);
5940
1250
 }
5941
1251
 
5942
1252
 int i915_gem_freeze(struct drm_i915_private *dev_priv)
..
..
@@ -5952,11 +1262,7 @@
5952
1262
 int i915_gem_freeze_late(struct drm_i915_private *i915)
5953
1263
 {
5954
1264
 	struct drm_i915_gem_object *obj;
5955
-	struct list_head *phases[] = {
5956
-		&i915->mm.unbound_list,
5957
-		&i915->mm.bound_list,
5958
-		NULL
5959
-	}, **phase;
1265
+	intel_wakeref_t wakeref;
5960
1266
 
5961
1267
 	/*
5962
1268
 	 * Called just before we write the hibernation image.
..
..
@@ -5973,32 +1279,21 @@
5973
1279
 	 * the objects as well, see i915_gem_freeze()
5974
1280
 	 */
5975
1281
 
5976
-	i915_gem_shrink(i915, -1UL, NULL, I915_SHRINK_UNBOUND);
1282
+	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1283
+
1284
+	i915_gem_shrink(i915, -1UL, NULL, ~0);
5977
1285
 	i915_gem_drain_freed_objects(i915);
5978
1286
 
5979
-	mutex_lock(&i915->drm.struct_mutex);
5980
-	for (phase = phases; *phase; phase++) {
5981
-		list_for_each_entry(obj, *phase, mm.link)
5982
-			WARN_ON(i915_gem_object_set_to_cpu_domain(obj, true));
1287
+	list_for_each_entry(obj, &i915->mm.shrink_list, mm.link) {
1288
+		i915_gem_object_lock(obj, NULL);
1289
+		drm_WARN_ON(&i915->drm,
1290
+			    i915_gem_object_set_to_cpu_domain(obj, true));
1291
+		i915_gem_object_unlock(obj);
5983
1292
 	}
5984
-	mutex_unlock(&i915->drm.struct_mutex);
1293
+
1294
+	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
5985
1295
 
5986
1296
 	return 0;
5987
-}
5988
-
5989
-void i915_gem_release(struct drm_device *dev, struct drm_file *file)
5990
-{
5991
-	struct drm_i915_file_private *file_priv = file->driver_priv;
5992
-	struct i915_request *request;
5993
-
5994
-	/* Clean up our request list when the client is going away, so that
5995
-	 * later retire_requests won't dereference our soon-to-be-gone
5996
-	 * file_priv.
5997
-	 */
5998
-	spin_lock(&file_priv->mm.lock);
5999
-	list_for_each_entry(request, &file_priv->mm.request_list, client_link)
6000
-		request->file_priv = NULL;
6001
-	spin_unlock(&file_priv->mm.lock);
6002
1297
 }
6003
1298
 
6004
1299
 int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file)
..
..
@@ -6016,9 +1311,6 @@
6016
1311
 	file_priv->dev_priv = i915;
6017
1312
 	file_priv->file = file;
6018
1313
 
6019
-	spin_lock_init(&file_priv->mm.lock);
6020
-	INIT_LIST_HEAD(&file_priv->mm.request_list);
6021
-
6022
1314
 	file_priv->bsd_engine = -1;
6023
1315
 	file_priv->hang_timestamp = jiffies;
6024
1316
 
..
..
@@ -6029,311 +1321,59 @@
6029
1321
 	return ret;
6030
1322
 }
6031
1323
 
6032
-/**
6033
- * i915_gem_track_fb - update frontbuffer tracking
6034
- * @old: current GEM buffer for the frontbuffer slots
6035
- * @new: new GEM buffer for the frontbuffer slots
6036
- * @frontbuffer_bits: bitmask of frontbuffer slots
6037
- *
6038
- * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
6039
- * from @old and setting them in @new. Both @old and @new can be NULL.
6040
- */
6041
-void i915_gem_track_fb(struct drm_i915_gem_object *old,
6042
-		       struct drm_i915_gem_object *new,
6043
-		       unsigned frontbuffer_bits)
1324
+void i915_gem_ww_ctx_init(struct i915_gem_ww_ctx *ww, bool intr)
6044
1325
 {
6045
-	/* Control of individual bits within the mask are guarded by
6046
-	 * the owning plane->mutex, i.e. we can never see concurrent
6047
-	 * manipulation of individual bits. But since the bitfield as a whole
6048
-	 * is updated using RMW, we need to use atomics in order to update
6049
-	 * the bits.
6050
-	 */
6051
-	BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES >
6052
-		     sizeof(atomic_t) * BITS_PER_BYTE);
6053
-
6054
-	if (old) {
6055
-		WARN_ON(!(atomic_read(&old->frontbuffer_bits) & frontbuffer_bits));
6056
-		atomic_andnot(frontbuffer_bits, &old->frontbuffer_bits);
6057
-	}
6058
-
6059
-	if (new) {
6060
-		WARN_ON(atomic_read(&new->frontbuffer_bits) & frontbuffer_bits);
6061
-		atomic_or(frontbuffer_bits, &new->frontbuffer_bits);
6062
-	}
1326
+	ww_acquire_init(&ww->ctx, &reservation_ww_class);
1327
+	INIT_LIST_HEAD(&ww->obj_list);
1328
+	ww->intr = intr;
1329
+	ww->contended = NULL;
6063
1330
 }
6064
1331
 
6065
-/* Allocate a new GEM object and fill it with the supplied data */
6066
-struct drm_i915_gem_object *
6067
-i915_gem_object_create_from_data(struct drm_i915_private *dev_priv,
6068
-			         const void *data, size_t size)
1332
+static void i915_gem_ww_ctx_unlock_all(struct i915_gem_ww_ctx *ww)
6069
1333
 {
6070
1334
 	struct drm_i915_gem_object *obj;
6071
-	struct file *file;
6072
-	size_t offset;
6073
-	int err;
6074
1335
 
6075
-	obj = i915_gem_object_create(dev_priv, round_up(size, PAGE_SIZE));
6076
-	if (IS_ERR(obj))
6077
-		return obj;
6078
-
6079
-	GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU);
6080
-
6081
-	file = obj->base.filp;
6082
-	offset = 0;
6083
-	do {
6084
-		unsigned int len = min_t(typeof(size), size, PAGE_SIZE);
6085
-		struct page *page;
6086
-		void *pgdata, *vaddr;
6087
-
6088
-		err = pagecache_write_begin(file, file->f_mapping,
6089
-					    offset, len, 0,
6090
-					    &page, &pgdata);
6091
-		if (err < 0)
6092
-			goto fail;
6093
-
6094
-		vaddr = kmap(page);
6095
-		memcpy(vaddr, data, len);
6096
-		kunmap(page);
6097
-
6098
-		err = pagecache_write_end(file, file->f_mapping,
6099
-					  offset, len, len,
6100
-					  page, pgdata);
6101
-		if (err < 0)
6102
-			goto fail;
6103
-
6104
-		size -= len;
6105
-		data += len;
6106
-		offset += len;
6107
-	} while (size);
6108
-
6109
-	return obj;
6110
-
6111
-fail:
6112
-	i915_gem_object_put(obj);
6113
-	return ERR_PTR(err);
6114
-}
6115
-
6116
-struct scatterlist *
6117
-i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
6118
-		       unsigned int n,
6119
-		       unsigned int *offset)
6120
-{
6121
-	struct i915_gem_object_page_iter *iter = &obj->mm.get_page;
6122
-	struct scatterlist *sg;
6123
-	unsigned int idx, count;
6124
-
6125
-	might_sleep();
6126
-	GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
6127
-	GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
6128
-
6129
-	/* As we iterate forward through the sg, we record each entry in a
6130
-	 * radixtree for quick repeated (backwards) lookups. If we have seen
6131
-	 * this index previously, we will have an entry for it.
6132
-	 *
6133
-	 * Initial lookup is O(N), but this is amortized to O(1) for
6134
-	 * sequential page access (where each new request is consecutive
6135
-	 * to the previous one). Repeated lookups are O(lg(obj->base.size)),
6136
-	 * i.e. O(1) with a large constant!
6137
-	 */
6138
-	if (n < READ_ONCE(iter->sg_idx))
6139
-		goto lookup;
6140
-
6141
-	mutex_lock(&iter->lock);
6142
-
6143
-	/* We prefer to reuse the last sg so that repeated lookup of this
6144
-	 * (or the subsequent) sg are fast - comparing against the last
6145
-	 * sg is faster than going through the radixtree.
6146
-	 */
6147
-
6148
-	sg = iter->sg_pos;
6149
-	idx = iter->sg_idx;
6150
-	count = __sg_page_count(sg);
6151
-
6152
-	while (idx + count <= n) {
6153
-		unsigned long exception, i;
6154
-		int ret;
6155
-
6156
-		/* If we cannot allocate and insert this entry, or the
6157
-		 * individual pages from this range, cancel updating the
6158
-		 * sg_idx so that on this lookup we are forced to linearly
6159
-		 * scan onwards, but on future lookups we will try the
6160
-		 * insertion again (in which case we need to be careful of
6161
-		 * the error return reporting that we have already inserted
6162
-		 * this index).
6163
-		 */
6164
-		ret = radix_tree_insert(&iter->radix, idx, sg);
6165
-		if (ret && ret != -EEXIST)
6166
-			goto scan;
6167
-
6168
-		exception =
6169
-			RADIX_TREE_EXCEPTIONAL_ENTRY |
6170
-			idx << RADIX_TREE_EXCEPTIONAL_SHIFT;
6171
-		for (i = 1; i < count; i++) {
6172
-			ret = radix_tree_insert(&iter->radix, idx + i,
6173
-						(void *)exception);
6174
-			if (ret && ret != -EEXIST)
6175
-				goto scan;
6176
-		}
6177
-
6178
-		idx += count;
6179
-		sg = ____sg_next(sg);
6180
-		count = __sg_page_count(sg);
1336
+	while ((obj = list_first_entry_or_null(&ww->obj_list, struct drm_i915_gem_object, obj_link))) {
1337
+		list_del(&obj->obj_link);
1338
+		i915_gem_object_unlock(obj);
6181
1339
 	}
6182
-
6183
-scan:
6184
-	iter->sg_pos = sg;
6185
-	iter->sg_idx = idx;
6186
-
6187
-	mutex_unlock(&iter->lock);
6188
-
6189
-	if (unlikely(n < idx)) /* insertion completed by another thread */
6190
-		goto lookup;
6191
-
6192
-	/* In case we failed to insert the entry into the radixtree, we need
6193
-	 * to look beyond the current sg.
6194
-	 */
6195
-	while (idx + count <= n) {
6196
-		idx += count;
6197
-		sg = ____sg_next(sg);
6198
-		count = __sg_page_count(sg);
6199
-	}
6200
-
6201
-	*offset = n - idx;
6202
-	return sg;
6203
-
6204
-lookup:
6205
-	rcu_read_lock();
6206
-
6207
-	sg = radix_tree_lookup(&iter->radix, n);
6208
-	GEM_BUG_ON(!sg);
6209
-
6210
-	/* If this index is in the middle of multi-page sg entry,
6211
-	 * the radixtree will contain an exceptional entry that points
6212
-	 * to the start of that range. We will return the pointer to
6213
-	 * the base page and the offset of this page within the
6214
-	 * sg entry's range.
6215
-	 */
6216
-	*offset = 0;
6217
-	if (unlikely(radix_tree_exception(sg))) {
6218
-		unsigned long base =
6219
-			(unsigned long)sg >> RADIX_TREE_EXCEPTIONAL_SHIFT;
6220
-
6221
-		sg = radix_tree_lookup(&iter->radix, base);
6222
-		GEM_BUG_ON(!sg);
6223
-
6224
-		*offset = n - base;
6225
-	}
6226
-
6227
-	rcu_read_unlock();
6228
-
6229
-	return sg;
6230
1340
 }
6231
1341
 
6232
-struct page *
6233
-i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n)
1342
+void i915_gem_ww_unlock_single(struct drm_i915_gem_object *obj)
6234
1343
 {
6235
-	struct scatterlist *sg;
6236
-	unsigned int offset;
6237
-
6238
-	GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
6239
-
6240
-	sg = i915_gem_object_get_sg(obj, n, &offset);
6241
-	return nth_page(sg_page(sg), offset);
1344
+	list_del(&obj->obj_link);
1345
+	i915_gem_object_unlock(obj);
6242
1346
 }
6243
1347
 
6244
-/* Like i915_gem_object_get_page(), but mark the returned page dirty */
6245
-struct page *
6246
-i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
6247
-			       unsigned int n)
1348
+void i915_gem_ww_ctx_fini(struct i915_gem_ww_ctx *ww)
6248
1349
 {
6249
-	struct page *page;
6250
-
6251
-	page = i915_gem_object_get_page(obj, n);
6252
-	if (!obj->mm.dirty)
6253
-		set_page_dirty(page);
6254
-
6255
-	return page;
1350
+	i915_gem_ww_ctx_unlock_all(ww);
1351
+	WARN_ON(ww->contended);
1352
+	ww_acquire_fini(&ww->ctx);
6256
1353
 }
6257
1354
 
6258
-dma_addr_t
6259
-i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
6260
-				unsigned long n)
1355
+int __must_check i915_gem_ww_ctx_backoff(struct i915_gem_ww_ctx *ww)
6261
1356
 {
6262
-	struct scatterlist *sg;
6263
-	unsigned int offset;
1357
+	int ret = 0;
6264
1358
 
6265
-	sg = i915_gem_object_get_sg(obj, n, &offset);
6266
-	return sg_dma_address(sg) + (offset << PAGE_SHIFT);
6267
-}
6268
-
6269
-int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align)
6270
-{
6271
-	struct sg_table *pages;
6272
-	int err;
6273
-
6274
-	if (align > obj->base.size)
1359
+	if (WARN_ON(!ww->contended))
6275
1360
 		return -EINVAL;
6276
1361
 
6277
-	if (obj->ops == &i915_gem_phys_ops)
6278
-		return 0;
1362
+	i915_gem_ww_ctx_unlock_all(ww);
1363
+	if (ww->intr)
1364
+		ret = dma_resv_lock_slow_interruptible(ww->contended->base.resv, &ww->ctx);
1365
+	else
1366
+		dma_resv_lock_slow(ww->contended->base.resv, &ww->ctx);
6279
1367
 
6280
-	if (obj->ops != &i915_gem_object_ops)
6281
-		return -EINVAL;
1368
+	if (!ret)
1369
+		list_add_tail(&ww->contended->obj_link, &ww->obj_list);
6282
1370
 
6283
-	err = i915_gem_object_unbind(obj);
6284
-	if (err)
6285
-		return err;
1371
+	ww->contended = NULL;
6286
1372
 
6287
-	mutex_lock(&obj->mm.lock);
6288
-
6289
-	if (obj->mm.madv != I915_MADV_WILLNEED) {
6290
-		err = -EFAULT;
6291
-		goto err_unlock;
6292
-	}
6293
-
6294
-	if (obj->mm.quirked) {
6295
-		err = -EFAULT;
6296
-		goto err_unlock;
6297
-	}
6298
-
6299
-	if (obj->mm.mapping) {
6300
-		err = -EBUSY;
6301
-		goto err_unlock;
6302
-	}
6303
-
6304
-	pages = __i915_gem_object_unset_pages(obj);
6305
-
6306
-	obj->ops = &i915_gem_phys_ops;
6307
-
6308
-	err = ____i915_gem_object_get_pages(obj);
6309
-	if (err)
6310
-		goto err_xfer;
6311
-
6312
-	/* Perma-pin (until release) the physical set of pages */
6313
-	__i915_gem_object_pin_pages(obj);
6314
-
6315
-	if (!IS_ERR_OR_NULL(pages))
6316
-		i915_gem_object_ops.put_pages(obj, pages);
6317
-	mutex_unlock(&obj->mm.lock);
6318
-	return 0;
6319
-
6320
-err_xfer:
6321
-	obj->ops = &i915_gem_object_ops;
6322
-	if (!IS_ERR_OR_NULL(pages)) {
6323
-		unsigned int sg_page_sizes = i915_sg_page_sizes(pages->sgl);
6324
-
6325
-		__i915_gem_object_set_pages(obj, pages, sg_page_sizes);
6326
-	}
6327
-err_unlock:
6328
-	mutex_unlock(&obj->mm.lock);
6329
-	return err;
1373
+	return ret;
6330
1374
 }
6331
1375
 
6332
1376
 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
6333
-#include "selftests/scatterlist.c"
6334
1377
 #include "selftests/mock_gem_device.c"
6335
-#include "selftests/huge_gem_object.c"
6336
-#include "selftests/huge_pages.c"
6337
-#include "selftests/i915_gem_object.c"
6338
-#include "selftests/i915_gem_coherency.c"
1378
+#include "selftests/i915_gem.c"
6339
1379
 #endif