forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-05-10 9999e48639b3cecb08ffb37358bcba3b48161b29 
 kernel/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
....@@ -22,42 +22,57 @@
2222  *
2323  */
2424 
25
-#include "../i915_selftest.h"
25
+#include "gem/i915_gem_pm.h"
26
+#include "gem/selftests/igt_gem_utils.h"
27
+#include "gem/selftests/mock_context.h"
28
+#include "gt/intel_gt.h"
2629 
30
+#include "i915_selftest.h"
31
+
32
+#include "igt_flush_test.h"
2733 #include "lib_sw_fence.h"
28
-#include "mock_context.h"
2934 #include "mock_drm.h"
3035 #include "mock_gem_device.h"
3136 
32
-static int populate_ggtt(struct drm_i915_private *i915)
37
+static void quirk_add(struct drm_i915_gem_object *obj,
38
+		      struct list_head *objects)
39
+{
40
+	/* quirk is only for live tiled objects, use it to declare ownership */
41
+	GEM_BUG_ON(obj->mm.quirked);
42
+	obj->mm.quirked = true;
43
+	list_add(&obj->st_link, objects);
44
+}
45
+
46
+static int populate_ggtt(struct i915_ggtt *ggtt, struct list_head *objects)
3347 {
3448 	struct drm_i915_gem_object *obj;
35
-	u64 size;
49
+	unsigned long count;
3650 
37
-	for (size = 0;
38
-	     size + I915_GTT_PAGE_SIZE <= i915->ggtt.vm.total;
39
-	     size += I915_GTT_PAGE_SIZE) {
51
+	count = 0;
52
+	do {
4053 		struct i915_vma *vma;
4154 
42
-		obj = i915_gem_object_create_internal(i915, I915_GTT_PAGE_SIZE);
55
+		obj = i915_gem_object_create_internal(ggtt->vm.i915,
56
+						      I915_GTT_PAGE_SIZE);
4357 		if (IS_ERR(obj))
4458 			return PTR_ERR(obj);
4559 
4660 		vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
47
-		if (IS_ERR(vma))
61
+		if (IS_ERR(vma)) {
62
+			i915_gem_object_put(obj);
63
+			if (vma == ERR_PTR(-ENOSPC))
64
+				break;
65
+
4866 			return PTR_ERR(vma);
49
-	}
67
+		}
5068 
51
-	if (!list_empty(&i915->mm.unbound_list)) {
52
-		size = 0;
53
-		list_for_each_entry(obj, &i915->mm.unbound_list, mm.link)
54
-			size++;
69
+		quirk_add(obj, objects);
70
+		count++;
71
+	} while (1);
72
+	pr_debug("Filled GGTT with %lu pages [%llu total]\n",
73
+		 count, ggtt->vm.total / PAGE_SIZE);
5574 
56
-		pr_err("Found %lld objects unbound!\n", size);
57
-		return -EINVAL;
58
-	}
59
-
60
-	if (list_empty(&i915->ggtt.vm.inactive_list)) {
75
+	if (list_empty(&ggtt->vm.bound_list)) {
6176 		pr_err("No objects on the GGTT inactive list!\n");
6277 		return -EINVAL;
6378 	}
....@@ -65,61 +80,63 @@
6580 	return 0;
6681 }
6782 
68
-static void unpin_ggtt(struct drm_i915_private *i915)
83
+static void unpin_ggtt(struct i915_ggtt *ggtt)
6984 {
7085 	struct i915_vma *vma;
7186 
72
-	list_for_each_entry(vma, &i915->ggtt.vm.inactive_list, vm_link)
73
-		i915_vma_unpin(vma);
87
+	list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
88
+		if (vma->obj->mm.quirked)
89
+			i915_vma_unpin(vma);
7490 }
7591 
76
-static void cleanup_objects(struct drm_i915_private *i915)
92
+static void cleanup_objects(struct i915_ggtt *ggtt, struct list_head *list)
7793 {
7894 	struct drm_i915_gem_object *obj, *on;
7995 
80
-	list_for_each_entry_safe(obj, on, &i915->mm.unbound_list, mm.link)
96
+	list_for_each_entry_safe(obj, on, list, st_link) {
97
+		GEM_BUG_ON(!obj->mm.quirked);
98
+		obj->mm.quirked = false;
8199 		i915_gem_object_put(obj);
100
+	}
82101 
83
-	list_for_each_entry_safe(obj, on, &i915->mm.bound_list, mm.link)
84
-		i915_gem_object_put(obj);
85
-
86
-	mutex_unlock(&i915->drm.struct_mutex);
87
-
88
-	i915_gem_drain_freed_objects(i915);
89
-
90
-	mutex_lock(&i915->drm.struct_mutex);
102
+	i915_gem_drain_freed_objects(ggtt->vm.i915);
91103 }
92104 
93105 static int igt_evict_something(void *arg)
94106 {
95
-	struct drm_i915_private *i915 = arg;
96
-	struct i915_ggtt *ggtt = &i915->ggtt;
107
+	struct intel_gt *gt = arg;
108
+	struct i915_ggtt *ggtt = gt->ggtt;
109
+	LIST_HEAD(objects);
97110 	int err;
98111 
99112 	/* Fill the GGTT with pinned objects and try to evict one. */
100113 
101
-	err = populate_ggtt(i915);
114
+	err = populate_ggtt(ggtt, &objects);
102115 	if (err)
103116 		goto cleanup;
104117 
105118 	/* Everything is pinned, nothing should happen */
119
+	mutex_lock(&ggtt->vm.mutex);
106120 	err = i915_gem_evict_something(&ggtt->vm,
107121 				       I915_GTT_PAGE_SIZE, 0, 0,
108122 				       0, U64_MAX,
109123 				       0);
124
+	mutex_unlock(&ggtt->vm.mutex);
110125 	if (err != -ENOSPC) {
111126 		pr_err("i915_gem_evict_something failed on a full GGTT with err=%d\n",
112127 		       err);
113128 		goto cleanup;
114129 	}
115130 
116
-	unpin_ggtt(i915);
131
+	unpin_ggtt(ggtt);
117132 
118133 	/* Everything is unpinned, we should be able to evict something */
134
+	mutex_lock(&ggtt->vm.mutex);
119135 	err = i915_gem_evict_something(&ggtt->vm,
120136 				       I915_GTT_PAGE_SIZE, 0, 0,
121137 				       0, U64_MAX,
122138 				       0);
139
+	mutex_unlock(&ggtt->vm.mutex);
123140 	if (err) {
124141 		pr_err("i915_gem_evict_something failed on a full GGTT with err=%d\n",
125142 		       err);
....@@ -127,71 +144,80 @@
127144 	}
128145 
129146 cleanup:
130
-	cleanup_objects(i915);
147
+	cleanup_objects(ggtt, &objects);
131148 	return err;
132149 }
133150 
134151 static int igt_overcommit(void *arg)
135152 {
136
-	struct drm_i915_private *i915 = arg;
153
+	struct intel_gt *gt = arg;
154
+	struct i915_ggtt *ggtt = gt->ggtt;
137155 	struct drm_i915_gem_object *obj;
138156 	struct i915_vma *vma;
157
+	LIST_HEAD(objects);
139158 	int err;
140159 
141160 	/* Fill the GGTT with pinned objects and then try to pin one more.
142161 	 * We expect it to fail.
143162 	 */
144163 
145
-	err = populate_ggtt(i915);
164
+	err = populate_ggtt(ggtt, &objects);
146165 	if (err)
147166 		goto cleanup;
148167 
149
-	obj = i915_gem_object_create_internal(i915, I915_GTT_PAGE_SIZE);
168
+	obj = i915_gem_object_create_internal(gt->i915, I915_GTT_PAGE_SIZE);
150169 	if (IS_ERR(obj)) {
151170 		err = PTR_ERR(obj);
152171 		goto cleanup;
153172 	}
154173 
174
+	quirk_add(obj, &objects);
175
+
155176 	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
156
-	if (!IS_ERR(vma) || PTR_ERR(vma) != -ENOSPC) {
157
-		pr_err("Failed to evict+insert, i915_gem_object_ggtt_pin returned err=%d\n", (int)PTR_ERR(vma));
177
+	if (vma != ERR_PTR(-ENOSPC)) {
178
+		pr_err("Failed to evict+insert, i915_gem_object_ggtt_pin returned err=%d\n", (int)PTR_ERR_OR_ZERO(vma));
158179 		err = -EINVAL;
159180 		goto cleanup;
160181 	}
161182 
162183 cleanup:
163
-	cleanup_objects(i915);
184
+	cleanup_objects(ggtt, &objects);
164185 	return err;
165186 }
166187 
167188 static int igt_evict_for_vma(void *arg)
168189 {
169
-	struct drm_i915_private *i915 = arg;
170
-	struct i915_ggtt *ggtt = &i915->ggtt;
190
+	struct intel_gt *gt = arg;
191
+	struct i915_ggtt *ggtt = gt->ggtt;
171192 	struct drm_mm_node target = {
172193 		.start = 0,
173194 		.size = 4096,
174195 	};
196
+	LIST_HEAD(objects);
175197 	int err;
176198 
177199 	/* Fill the GGTT with pinned objects and try to evict a range. */
178200 
179
-	err = populate_ggtt(i915);
201
+	err = populate_ggtt(ggtt, &objects);
180202 	if (err)
181203 		goto cleanup;
182204 
183205 	/* Everything is pinned, nothing should happen */
206
+	mutex_lock(&ggtt->vm.mutex);
184207 	err = i915_gem_evict_for_node(&ggtt->vm, &target, 0);
208
+	mutex_unlock(&ggtt->vm.mutex);
185209 	if (err != -ENOSPC) {
186210 		pr_err("i915_gem_evict_for_node on a full GGTT returned err=%d\n",
187211 		       err);
188212 		goto cleanup;
189213 	}
190214 
191
-	unpin_ggtt(i915);
215
+	unpin_ggtt(ggtt);
192216 
193217 	/* Everything is unpinned, we should be able to evict the node */
218
+	mutex_lock(&ggtt->vm.mutex);
194219 	err = i915_gem_evict_for_node(&ggtt->vm, &target, 0);
220
+	mutex_unlock(&ggtt->vm.mutex);
195221 	if (err) {
196222 		pr_err("i915_gem_evict_for_node returned err=%d\n",
197223 		       err);
....@@ -199,7 +225,7 @@
199225 	}
200226 
201227 cleanup:
202
-	cleanup_objects(i915);
228
+	cleanup_objects(ggtt, &objects);
203229 	return err;
204230 }
205231 
....@@ -212,8 +238,8 @@
212238 
213239 static int igt_evict_for_cache_color(void *arg)
214240 {
215
-	struct drm_i915_private *i915 = arg;
216
-	struct i915_ggtt *ggtt = &i915->ggtt;
241
+	struct intel_gt *gt = arg;
242
+	struct i915_ggtt *ggtt = gt->ggtt;
217243 	const unsigned long flags = PIN_OFFSET_FIXED;
218244 	struct drm_mm_node target = {
219245 		.start = I915_GTT_PAGE_SIZE * 2,
....@@ -222,21 +248,25 @@
222248 	};
223249 	struct drm_i915_gem_object *obj;
224250 	struct i915_vma *vma;
251
+	LIST_HEAD(objects);
225252 	int err;
226253 
227
-	/* Currently the use of color_adjust is limited to cache domains within
228
-	 * the ggtt, and so the presence of mm.color_adjust is assumed to be
229
-	 * i915_gtt_color_adjust throughout our driver, so using a mock color
230
-	 * adjust will work just fine for our purposes.
254
+	/*
255
+	 * Currently the use of color_adjust for the GGTT is limited to cache
256
+	 * coloring and guard pages, and so the presence of mm.color_adjust for
257
+	 * the GGTT is assumed to be i915_ggtt_color_adjust, hence using a mock
258
+	 * color adjust will work just fine for our purposes.
231259 	 */
232260 	ggtt->vm.mm.color_adjust = mock_color_adjust;
261
+	GEM_BUG_ON(!i915_vm_has_cache_coloring(&ggtt->vm));
233262 
234
-	obj = i915_gem_object_create_internal(i915, I915_GTT_PAGE_SIZE);
263
+	obj = i915_gem_object_create_internal(gt->i915, I915_GTT_PAGE_SIZE);
235264 	if (IS_ERR(obj)) {
236265 		err = PTR_ERR(obj);
237266 		goto cleanup;
238267 	}
239
-	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
268
+	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
269
+	quirk_add(obj, &objects);
240270 
241271 	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
242272 				       I915_GTT_PAGE_SIZE | flags);
....@@ -246,12 +276,13 @@
246276 		goto cleanup;
247277 	}
248278 
249
-	obj = i915_gem_object_create_internal(i915, I915_GTT_PAGE_SIZE);
279
+	obj = i915_gem_object_create_internal(gt->i915, I915_GTT_PAGE_SIZE);
250280 	if (IS_ERR(obj)) {
251281 		err = PTR_ERR(obj);
252282 		goto cleanup;
253283 	}
254
-	i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
284
+	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
285
+	quirk_add(obj, &objects);
255286 
256287 	/* Neighbouring; same colour - should fit */
257288 	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
....@@ -265,7 +296,9 @@
265296 	i915_vma_unpin(vma);
266297 
267298 	/* Remove just the second vma */
299
+	mutex_lock(&ggtt->vm.mutex);
268300 	err = i915_gem_evict_for_node(&ggtt->vm, &target, 0);
301
+	mutex_unlock(&ggtt->vm.mutex);
269302 	if (err) {
270303 		pr_err("[0]i915_gem_evict_for_node returned err=%d\n", err);
271304 		goto cleanup;
....@@ -276,7 +309,9 @@
276309 	 */
277310 	target.color = I915_CACHE_L3_LLC;
278311 
312
+	mutex_lock(&ggtt->vm.mutex);
279313 	err = i915_gem_evict_for_node(&ggtt->vm, &target, 0);
314
+	mutex_unlock(&ggtt->vm.mutex);
280315 	if (!err) {
281316 		pr_err("[1]i915_gem_evict_for_node returned err=%d\n", err);
282317 		err = -EINVAL;
....@@ -286,35 +321,40 @@
286321 	err = 0;
287322 
288323 cleanup:
289
-	unpin_ggtt(i915);
290
-	cleanup_objects(i915);
324
+	unpin_ggtt(ggtt);
325
+	cleanup_objects(ggtt, &objects);
291326 	ggtt->vm.mm.color_adjust = NULL;
292327 	return err;
293328 }
294329 
295330 static int igt_evict_vm(void *arg)
296331 {
297
-	struct drm_i915_private *i915 = arg;
298
-	struct i915_ggtt *ggtt = &i915->ggtt;
332
+	struct intel_gt *gt = arg;
333
+	struct i915_ggtt *ggtt = gt->ggtt;
334
+	LIST_HEAD(objects);
299335 	int err;
300336 
301337 	/* Fill the GGTT with pinned objects and try to evict everything. */
302338 
303
-	err = populate_ggtt(i915);
339
+	err = populate_ggtt(ggtt, &objects);
304340 	if (err)
305341 		goto cleanup;
306342 
307343 	/* Everything is pinned, nothing should happen */
344
+	mutex_lock(&ggtt->vm.mutex);
308345 	err = i915_gem_evict_vm(&ggtt->vm);
346
+	mutex_unlock(&ggtt->vm.mutex);
309347 	if (err) {
310348 		pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n",
311349 		       err);
312350 		goto cleanup;
313351 	}
314352 
315
-	unpin_ggtt(i915);
353
+	unpin_ggtt(ggtt);
316354 
355
+	mutex_lock(&ggtt->vm.mutex);
317356 	err = i915_gem_evict_vm(&ggtt->vm);
357
+	mutex_unlock(&ggtt->vm.mutex);
318358 	if (err) {
319359 		pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n",
320360 		       err);
....@@ -322,20 +362,23 @@
322362 	}
323363 
324364 cleanup:
325
-	cleanup_objects(i915);
365
+	cleanup_objects(ggtt, &objects);
326366 	return err;
327367 }
328368 
329369 static int igt_evict_contexts(void *arg)
330370 {
331371 	const u64 PRETEND_GGTT_SIZE = 16ull << 20;
332
-	struct drm_i915_private *i915 = arg;
372
+	struct intel_gt *gt = arg;
373
+	struct i915_ggtt *ggtt = gt->ggtt;
374
+	struct drm_i915_private *i915 = gt->i915;
333375 	struct intel_engine_cs *engine;
334376 	enum intel_engine_id id;
335377 	struct reserved {
336378 		struct drm_mm_node node;
337379 		struct reserved *next;
338380 	} *reserved = NULL;
381
+	intel_wakeref_t wakeref;
339382 	struct drm_mm_node hole;
340383 	unsigned long count;
341384 	int err;
....@@ -351,17 +394,17 @@
351394 	 * where the GTT space of the request is separate from the GGTT
352395 	 * allocation required to build the request.
353396 	 */
354
-	if (!USES_FULL_PPGTT(i915))
397
+	if (!HAS_FULL_PPGTT(i915))
355398 		return 0;
356399 
357
-	mutex_lock(&i915->drm.struct_mutex);
358
-	intel_runtime_pm_get(i915);
400
+	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
359401 
360402 	/* Reserve a block so that we know we have enough to fit a few rq */
361403 	memset(&hole, 0, sizeof(hole));
362
-	err = i915_gem_gtt_insert(&i915->ggtt.vm, &hole,
404
+	mutex_lock(&ggtt->vm.mutex);
405
+	err = i915_gem_gtt_insert(&ggtt->vm, &hole,
363406 				  PRETEND_GGTT_SIZE, 0, I915_COLOR_UNEVICTABLE,
364
-				  0, i915->ggtt.vm.total,
407
+				  0, ggtt->vm.total,
365408 				  PIN_NOEVICT);
366409 	if (err)
367410 		goto out_locked;
....@@ -371,15 +414,17 @@
371414 	do {
372415 		struct reserved *r;
373416 
417
+		mutex_unlock(&ggtt->vm.mutex);
374418 		r = kcalloc(1, sizeof(*r), GFP_KERNEL);
419
+		mutex_lock(&ggtt->vm.mutex);
375420 		if (!r) {
376421 			err = -ENOMEM;
377422 			goto out_locked;
378423 		}
379424 
380
-		if (i915_gem_gtt_insert(&i915->ggtt.vm, &r->node,
425
+		if (i915_gem_gtt_insert(&ggtt->vm, &r->node,
381426 					1ul << 20, 0, I915_COLOR_UNEVICTABLE,
382
-					0, i915->ggtt.vm.total,
427
+					0, ggtt->vm.total,
383428 					PIN_NOEVICT)) {
384429 			kfree(r);
385430 			break;
....@@ -391,39 +436,40 @@
391436 		count++;
392437 	} while (1);
393438 	drm_mm_remove_node(&hole);
394
-	mutex_unlock(&i915->drm.struct_mutex);
439
+	mutex_unlock(&ggtt->vm.mutex);
395440 	pr_info("Filled GGTT with %lu 1MiB nodes\n", count);
396441 
397442 	/* Overfill the GGTT with context objects and so try to evict one. */
398
-	for_each_engine(engine, i915, id) {
443
+	for_each_engine(engine, gt, id) {
399444 		struct i915_sw_fence fence;
400
-		struct drm_file *file;
445
+		struct file *file;
401446 
402447 		file = mock_file(i915);
403
-		if (IS_ERR(file))
404
-			return PTR_ERR(file);
448
+		if (IS_ERR(file)) {
449
+			err = PTR_ERR(file);
450
+			break;
451
+		}
405452 
406453 		count = 0;
407
-		mutex_lock(&i915->drm.struct_mutex);
408454 		onstack_fence_init(&fence);
409455 		do {
410456 			struct i915_request *rq;
411457 			struct i915_gem_context *ctx;
412458 
413459 			ctx = live_context(i915, file);
414
-			if (!ctx)
460
+			if (IS_ERR(ctx))
415461 				break;
416462 
417463 			/* We will need some GGTT space for the rq's context */
418464 			igt_evict_ctl.fail_if_busy = true;
419
-			rq = i915_request_alloc(engine, ctx);
465
+			rq = igt_request_alloc(ctx, engine);
420466 			igt_evict_ctl.fail_if_busy = false;
421467 
422468 			if (IS_ERR(rq)) {
423469 				/* When full, fail_if_busy will trigger EBUSY */
424470 				if (PTR_ERR(rq) != -EBUSY) {
425
-					pr_err("Unexpected error from request alloc (ctx hw id %u, on %s): %d\n",
426
-					       ctx->hw_id, engine->name,
471
+					pr_err("Unexpected error from request alloc (on %s): %d\n",
472
+					       engine->name,
427473 					       (int)PTR_ERR(rq));
428474 					err = PTR_ERR(rq);
429475 				}
....@@ -441,19 +487,19 @@
441487 			count++;
442488 			err = 0;
443489 		} while(1);
444
-		mutex_unlock(&i915->drm.struct_mutex);
445
-
446490 		onstack_fence_fini(&fence);
447491 		pr_info("Submitted %lu contexts/requests on %s\n",
448492 			count, engine->name);
449493 
450
-		mock_file_free(i915, file);
494
+		fput(file);
451495 		if (err)
452496 			break;
453497 	}
454498 
455
-	mutex_lock(&i915->drm.struct_mutex);
499
+	mutex_lock(&ggtt->vm.mutex);
456500 out_locked:
501
+	if (igt_flush_test(i915))
502
+		err = -EIO;
457503 	while (reserved) {
458504 		struct reserved *next = reserved->next;
459505 
....@@ -464,8 +510,8 @@
464510 	}
465511 	if (drm_mm_node_allocated(&hole))
466512 		drm_mm_remove_node(&hole);
467
-	intel_runtime_pm_put(i915);
468
-	mutex_unlock(&i915->drm.struct_mutex);
513
+	mutex_unlock(&ggtt->vm.mutex);
514
+	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
469515 
470516 	return err;
471517 }
....@@ -480,17 +526,17 @@
480526 		SUBTEST(igt_overcommit),
481527 	};
482528 	struct drm_i915_private *i915;
483
-	int err;
529
+	intel_wakeref_t wakeref;
530
+	int err = 0;
484531 
485532 	i915 = mock_gem_device();
486533 	if (!i915)
487534 		return -ENOMEM;
488535 
489
-	mutex_lock(&i915->drm.struct_mutex);
490
-	err = i915_subtests(tests, i915);
491
-	mutex_unlock(&i915->drm.struct_mutex);
536
+	with_intel_runtime_pm(&i915->runtime_pm, wakeref)
537
+		err = i915_subtests(tests, &i915->gt);
492538 
493
-	drm_dev_put(&i915->drm);
539
+	mock_destroy_device(i915);
494540 	return err;
495541 }
496542 
....@@ -500,8 +546,8 @@
500546 		SUBTEST(igt_evict_contexts),
501547 	};
502548 
503
-	if (i915_terminally_wedged(&i915->gpu_error))
549
+	if (intel_gt_is_wedged(&i915->gt))
504550 		return 0;
505551 
506
-	return i915_subtests(tests, i915);
552
+	return intel_gt_live_subtests(tests, &i915->gt);
507553 }