forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-01-31 f70575805708cabdedea7498aaa3f710fde4d920 
 kernel/drivers/gpu/drm/i915/i915_gem_evict.c
....@@ -26,56 +26,26 @@
2626  *
2727  */
2828 
29
-#include <drm/drmP.h>
30
-#include <drm/i915_drm.h>
29
+#include "gem/i915_gem_context.h"
30
+#include "gt/intel_gt_requests.h"
3131 
3232 #include "i915_drv.h"
33
-#include "intel_drv.h"
3433 #include "i915_trace.h"
3534 
3635 I915_SELFTEST_DECLARE(static struct igt_evict_ctl {
3736 	bool fail_if_busy:1;
3837 } igt_evict_ctl;)
3938 
40
-static bool ggtt_is_idle(struct drm_i915_private *i915)
39
+static int ggtt_flush(struct intel_gt *gt)
4140 {
42
-       struct intel_engine_cs *engine;
43
-       enum intel_engine_id id;
44
-
45
-       if (i915->gt.active_requests)
46
-	       return false;
47
-
48
-       for_each_engine(engine, i915, id) {
49
-	       if (!intel_engine_has_kernel_context(engine))
50
-		       return false;
51
-       }
52
-
53
-       return true;
54
-}
55
-
56
-static int ggtt_flush(struct drm_i915_private *i915)
57
-{
58
-	int err;
59
-
60
-	/* Not everything in the GGTT is tracked via vma (otherwise we
41
+	/*
42
+	 * Not everything in the GGTT is tracked via vma (otherwise we
6143 	 * could evict as required with minimal stalling) so we are forced
6244 	 * to idle the GPU and explicitly retire outstanding requests in
6345 	 * the hopes that we can then remove contexts and the like only
6446 	 * bound by their active reference.
6547 	 */
66
-	err = i915_gem_switch_to_kernel_context(i915);
67
-	if (err)
68
-		return err;
69
-
70
-	err = i915_gem_wait_for_idle(i915,
71
-				     I915_WAIT_INTERRUPTIBLE |
72
-				     I915_WAIT_LOCKED,
73
-				     MAX_SCHEDULE_TIMEOUT);
74
-	if (err)
75
-		return err;
76
-
77
-	GEM_BUG_ON(!ggtt_is_idle(i915));
78
-	return 0;
48
+	return intel_gt_wait_for_idle(gt, MAX_SCHEDULE_TIMEOUT);
7949 }
8050 
8151 static bool
....@@ -87,9 +57,6 @@
8757 	if (i915_vma_is_pinned(vma))
8858 		return false;
8959 
90
-	if (flags & PIN_NONFAULT && i915_vma_has_userfault(vma))
91
-		return false;
92
-
9360 	list_add(&vma->evict_link, unwind);
9461 	return drm_mm_scan_add_block(scan, &vma->node);
9562 }
....@@ -99,7 +66,7 @@
9966  * @vm: address space to evict from
10067  * @min_size: size of the desired free space
10168  * @alignment: alignment constraint of the desired free space
102
- * @cache_level: cache_level for the desired space
69
+ * @color: color for the desired space
10370  * @start: start (inclusive) of the range from which to evict objects
10471  * @end: end (exclusive) of the range from which to evict objects
10572  * @flags: additional flags to control the eviction algorithm
....@@ -120,38 +87,31 @@
12087 int
12188 i915_gem_evict_something(struct i915_address_space *vm,
12289 			 u64 min_size, u64 alignment,
123
-			 unsigned cache_level,
90
+			 unsigned long color,
12491 			 u64 start, u64 end,
12592 			 unsigned flags)
12693 {
127
-	struct drm_i915_private *dev_priv = vm->i915;
12894 	struct drm_mm_scan scan;
12995 	struct list_head eviction_list;
130
-	struct list_head *phases[] = {
131
-		&vm->inactive_list,
132
-		&vm->active_list,
133
-		NULL,
134
-	}, **phase;
13596 	struct i915_vma *vma, *next;
13697 	struct drm_mm_node *node;
13798 	enum drm_mm_insert_mode mode;
99
+	struct i915_vma *active;
138100 	int ret;
139101 
140
-	lockdep_assert_held(&vm->i915->drm.struct_mutex);
102
+	lockdep_assert_held(&vm->mutex);
141103 	trace_i915_gem_evict(vm, min_size, alignment, flags);
142104 
143105 	/*
144
-	 * The goal is to evict objects and amalgamate space in LRU order.
145
-	 * The oldest idle objects reside on the inactive list, which is in
146
-	 * retirement order. The next objects to retire are those in flight,
147
-	 * on the active list, again in retirement order.
106
+	 * The goal is to evict objects and amalgamate space in rough LRU order.
107
+	 * Since both active and inactive objects reside on the same list,
108
+	 * in a mix of creation and last scanned order, as we process the list
109
+	 * we sort it into inactive/active, which keeps the active portion
110
+	 * in a rough MRU order.
148111 	 *
149112 	 * The retirement sequence is thus:
150
-	 *   1. Inactive objects (already retired)
151
-	 *   2. Active objects (will stall on unbinding)
152
-	 *
153
-	 * On each list, the oldest objects lie at the HEAD with the freshest
154
-	 * object on the TAIL.
113
+	 *   1. Inactive objects (already retired, random order)
114
+	 *   2. Active objects (will stall on unbinding, oldest scanned first)
155115 	 */
156116 	mode = DRM_MM_INSERT_BEST;
157117 	if (flags & PIN_HIGH)
....@@ -159,28 +119,48 @@
159119 	if (flags & PIN_MAPPABLE)
160120 		mode = DRM_MM_INSERT_LOW;
161121 	drm_mm_scan_init_with_range(&scan, &vm->mm,
162
-				    min_size, alignment, cache_level,
122
+				    min_size, alignment, color,
163123 				    start, end, mode);
164124 
165
-	/*
166
-	 * Retire before we search the active list. Although we have
167
-	 * reasonable accuracy in our retirement lists, we may have
168
-	 * a stray pin (preventing eviction) that can only be resolved by
169
-	 * retiring.
170
-	 */
171
-	if (!(flags & PIN_NONBLOCK))
172
-		i915_retire_requests(dev_priv);
173
-	else
174
-		phases[1] = NULL;
125
+	intel_gt_retire_requests(vm->gt);
175126 
176127 search_again:
128
+	active = NULL;
177129 	INIT_LIST_HEAD(&eviction_list);
178
-	phase = phases;
179
-	do {
180
-		list_for_each_entry(vma, *phase, vm_link)
181
-			if (mark_free(&scan, vma, flags, &eviction_list))
182
-				goto found;
183
-	} while (*++phase);
130
+	list_for_each_entry_safe(vma, next, &vm->bound_list, vm_link) {
131
+		if (vma == active) { /* now seen this vma twice */
132
+			if (flags & PIN_NONBLOCK)
133
+				break;
134
+
135
+			active = ERR_PTR(-EAGAIN);
136
+		}
137
+
138
+		/*
139
+		 * We keep this list in a rough least-recently scanned order
140
+		 * of active elements (inactive elements are cheap to reap).
141
+		 * New entries are added to the end, and we move anything we
142
+		 * scan to the end. The assumption is that the working set
143
+		 * of applications is either steady state (and thanks to the
144
+		 * userspace bo cache it almost always is) or volatile and
145
+		 * frequently replaced after a frame, which are self-evicting!
146
+		 * Given that assumption, the MRU order of the scan list is
147
+		 * fairly static, and keeping it in least-recently scan order
148
+		 * is suitable.
149
+		 *
150
+		 * To notice when we complete one full cycle, we record the
151
+		 * first active element seen, before moving it to the tail.
152
+		 */
153
+		if (active != ERR_PTR(-EAGAIN) && i915_vma_is_active(vma)) {
154
+			if (!active)
155
+				active = vma;
156
+
157
+			list_move_tail(&vma->vm_link, &vm->bound_list);
158
+			continue;
159
+		}
160
+
161
+		if (mark_free(&scan, vma, flags, &eviction_list))
162
+			goto found;
163
+	}
184164 
185165 	/* Nothing found, clean up and bail out! */
186166 	list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
....@@ -210,24 +190,17 @@
210190 	 * us a termination condition, when the last retired context is
211191 	 * the kernel's there is no more we can evict.
212192 	 */
213
-	if (!ggtt_is_idle(dev_priv)) {
214
-		if (I915_SELFTEST_ONLY(igt_evict_ctl.fail_if_busy))
215
-			return -EBUSY;
193
+	if (I915_SELFTEST_ONLY(igt_evict_ctl.fail_if_busy))
194
+		return -EBUSY;
216195 
217
-		ret = ggtt_flush(dev_priv);
218
-		if (ret)
219
-			return ret;
196
+	ret = ggtt_flush(vm->gt);
197
+	if (ret)
198
+		return ret;
220199 
221
-		cond_resched();
222
-		goto search_again;
223
-	}
200
+	cond_resched();
224201 
225
-	/*
226
-	 * If we still have pending pageflip completions, drop
227
-	 * back to userspace to give our workqueues time to
228
-	 * acquire our locks and unpin the old scanouts.
229
-	 */
230
-	return intel_has_pending_fb_unpin(dev_priv) ? -EAGAIN : -ENOSPC;
202
+	flags |= PIN_NONBLOCK;
203
+	goto search_again;
231204 
232205 found:
233206 	/* drm_mm doesn't allow any other other operations while
....@@ -248,12 +221,17 @@
248221 	list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
249222 		__i915_vma_unpin(vma);
250223 		if (ret == 0)
251
-			ret = i915_vma_unbind(vma);
224
+			ret = __i915_vma_unbind(vma);
252225 	}
253226 
254227 	while (ret == 0 && (node = drm_mm_scan_color_evict(&scan))) {
255228 		vma = container_of(node, struct i915_vma, node);
256
-		ret = i915_vma_unbind(vma);
229
+
230
+		/* If we find any non-objects (!vma), we cannot evict them */
231
+		if (vma->node.color != I915_COLOR_UNEVICTABLE)
232
+			ret = __i915_vma_unbind(vma);
233
+		else
234
+			ret = -ENOSPC; /* XXX search failed, try again? */
257235 	}
258236 
259237 	return ret;
....@@ -279,25 +257,23 @@
279257 	u64 start = target->start;
280258 	u64 end = start + target->size;
281259 	struct i915_vma *vma, *next;
282
-	bool check_color;
283260 	int ret = 0;
284261 
285
-	lockdep_assert_held(&vm->i915->drm.struct_mutex);
262
+	lockdep_assert_held(&vm->mutex);
286263 	GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
287264 	GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
288265 
289266 	trace_i915_gem_evict_node(vm, target, flags);
290267 
291
-	/* Retire before we search the active list. Although we have
268
+	/*
269
+	 * Retire before we search the active list. Although we have
292270 	 * reasonable accuracy in our retirement lists, we may have
293271 	 * a stray pin (preventing eviction) that can only be resolved by
294272 	 * retiring.
295273 	 */
296
-	if (!(flags & PIN_NONBLOCK))
297
-		i915_retire_requests(vm->i915);
274
+	intel_gt_retire_requests(vm->gt);
298275 
299
-	check_color = vm->mm.color_adjust;
300
-	if (check_color) {
276
+	if (i915_vm_has_cache_coloring(vm)) {
301277 		/* Expand search to cover neighbouring guard pages (or lack!) */
302278 		if (start)
303279 			start -= I915_GTT_PAGE_SIZE;
....@@ -314,16 +290,17 @@
314290 			break;
315291 		}
316292 
317
-		GEM_BUG_ON(!node->allocated);
293
+		GEM_BUG_ON(!drm_mm_node_allocated(node));
318294 		vma = container_of(node, typeof(*vma), node);
319295 
320
-		/* If we are using coloring to insert guard pages between
296
+		/*
297
+		 * If we are using coloring to insert guard pages between
321298 		 * different cache domains within the address space, we have
322299 		 * to check whether the objects on either side of our range
323300 		 * abutt and conflict. If they are in conflict, then we evict
324301 		 * those as well to make room for our guard pages.
325302 		 */
326
-		if (check_color) {
303
+		if (i915_vm_has_cache_coloring(vm)) {
327304 			if (node->start + node->size == target->start) {
328305 				if (node->color == target->color)
329306 					continue;
....@@ -334,27 +311,18 @@
334311 			}
335312 		}
336313 
337
-		if (flags & PIN_NONBLOCK &&
338
-		    (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))) {
339
-			ret = -ENOSPC;
340
-			break;
341
-		}
342
-
343
-		if (flags & PIN_NONFAULT && i915_vma_has_userfault(vma)) {
344
-			ret = -ENOSPC;
345
-			break;
346
-		}
347
-
348
-		/* Overlap of objects in the same batch? */
349314 		if (i915_vma_is_pinned(vma)) {
350315 			ret = -ENOSPC;
351
-			if (vma->exec_flags &&
352
-			    *vma->exec_flags & EXEC_OBJECT_PINNED)
353
-				ret = -EINVAL;
354316 			break;
355317 		}
356318 
357
-		/* Never show fear in the face of dragons!
319
+		if (flags & PIN_NONBLOCK && i915_vma_is_active(vma)) {
320
+			ret = -ENOSPC;
321
+			break;
322
+		}
323
+
324
+		/*
325
+		 * Never show fear in the face of dragons!
358326 		 *
359327 		 * We cannot directly remove this node from within this
360328 		 * iterator and as with i915_gem_evict_something() we employ
....@@ -369,7 +337,7 @@
369337 	list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
370338 		__i915_vma_unpin(vma);
371339 		if (ret == 0)
372
-			ret = i915_vma_unbind(vma);
340
+			ret = __i915_vma_unbind(vma);
373341 	}
374342 
375343 	return ret;
....@@ -389,16 +357,9 @@
389357  */
390358 int i915_gem_evict_vm(struct i915_address_space *vm)
391359 {
392
-	struct list_head *phases[] = {
393
-		&vm->inactive_list,
394
-		&vm->active_list,
395
-		NULL
396
-	}, **phase;
397
-	struct list_head eviction_list;
398
-	struct i915_vma *vma, *next;
399
-	int ret;
360
+	int ret = 0;
400361 
401
-	lockdep_assert_held(&vm->i915->drm.struct_mutex);
362
+	lockdep_assert_held(&vm->mutex);
402363 	trace_i915_gem_evict_vm(vm);
403364 
404365 	/* Switch back to the default context in order to unpin
....@@ -407,29 +368,35 @@
407368 	 * switch otherwise is ineffective.
408369 	 */
409370 	if (i915_is_ggtt(vm)) {
410
-		ret = ggtt_flush(vm->i915);
371
+		ret = ggtt_flush(vm->gt);
411372 		if (ret)
412373 			return ret;
413374 	}
414375 
415
-	INIT_LIST_HEAD(&eviction_list);
416
-	phase = phases;
417376 	do {
418
-		list_for_each_entry(vma, *phase, vm_link) {
377
+		struct i915_vma *vma, *vn;
378
+		LIST_HEAD(eviction_list);
379
+
380
+		list_for_each_entry(vma, &vm->bound_list, vm_link) {
419381 			if (i915_vma_is_pinned(vma))
420382 				continue;
421383 
422384 			__i915_vma_pin(vma);
423385 			list_add(&vma->evict_link, &eviction_list);
424386 		}
425
-	} while (*++phase);
387
+		if (list_empty(&eviction_list))
388
+			break;
426389 
427
-	ret = 0;
428
-	list_for_each_entry_safe(vma, next, &eviction_list, evict_link) {
429
-		__i915_vma_unpin(vma);
430
-		if (ret == 0)
431
-			ret = i915_vma_unbind(vma);
432
-	}
390
+		ret = 0;
391
+		list_for_each_entry_safe(vma, vn, &eviction_list, evict_link) {
392
+			__i915_vma_unpin(vma);
393
+			if (ret == 0)
394
+				ret = __i915_vma_unbind(vma);
395
+			if (ret != -EINTR) /* "Get me out of here!" */
396
+				ret = 0;
397
+		}
398
+	} while (ret == 0);
399
+
433400 	return ret;
434401 }
435402