~hc/RK356X_SDK_RELEASE.git

..	..	@@ -25,25 +25,21 @@
25	25	#include <linux/list_sort.h>
26	26	#include <linux/prime_numbers.h>
27	27
28		-#include "../i915_selftest.h"
29		-#include "i915_random.h"
	28	+#include "gem/i915_gem_context.h"
	29	+#include "gem/selftests/mock_context.h"
	30	+#include "gt/intel_context.h"
30	31
31		-#include "mock_context.h"
	32	+#include "i915_random.h"
	33	+#include "i915_selftest.h"
	34	+
32	35	#include "mock_drm.h"
33	36	#include "mock_gem_device.h"
	37	+#include "mock_gtt.h"
	38	+#include "igt_flush_test.h"
34	39
35	40	static void cleanup_freed_objects(struct drm_i915_private *i915)
36	41	{
37		- /*
38		- * As we may hold onto the struct_mutex for inordinate lengths of
39		- * time, the NMI khungtaskd detector may fire for the free objects
40		- * worker.
41		- */
42		- mutex_unlock(&i915->drm.struct_mutex);
43		-
44	42	i915_gem_drain_freed_objects(i915);
45		-
46		- mutex_lock(&i915->drm.struct_mutex);
47	43	}
48	44
49	45	static void fake_free_pages(struct drm_i915_gem_object *obj,
..	..	@@ -87,8 +83,6 @@
87	83	}
88	84	GEM_BUG_ON(rem);
89	85
90		- obj->mm.madv = I915_MADV_DONTNEED;
91		-
92	86	__i915_gem_object_set_pages(obj, pages, sg_page_sizes);
93	87
94	88	return 0;
..	..	@@ -100,10 +94,10 @@
100	94	{
101	95	fake_free_pages(obj, pages);
102	96	obj->mm.dirty = false;
103		- obj->mm.madv = I915_MADV_WILLNEED;
104	97	}
105	98
106	99	static const struct drm_i915_gem_object_ops fake_ops = {
	100	+ .name = "fake-gem",
107	101	.flags = I915_GEM_OBJECT_IS_SHRINKABLE,
108	102	.get_pages = fake_get_pages,
109	103	.put_pages = fake_put_pages,
..	..	@@ -112,6 +106,7 @@
112	106	static struct drm_i915_gem_object *
113	107	fake_dma_object(struct drm_i915_private *i915, u64 size)
114	108	{
	109	+ static struct lock_class_key lock_class;
115	110	struct drm_i915_gem_object *obj;
116	111
117	112	GEM_BUG_ON(!size);
..	..	@@ -120,12 +115,14 @@
120	115	if (overflows_type(size, obj->base.size))
121	116	return ERR_PTR(-E2BIG);
122	117
123		- obj = i915_gem_object_alloc(i915);
	118	+ obj = i915_gem_object_alloc();
124	119	if (!obj)
125	120	goto err;
126	121
127	122	drm_gem_private_object_init(&i915->drm, &obj->base, size);
128		- i915_gem_object_init(obj, &fake_ops);
	123	+ i915_gem_object_init(obj, &fake_ops, &lock_class);
	124	+
	125	+ i915_gem_object_set_volatile(obj);
129	126
130	127	obj->write_domain = I915_GEM_DOMAIN_CPU;
131	128	obj->read_domains = I915_GEM_DOMAIN_CPU;
..	..	@@ -147,16 +144,16 @@
147	144	static int igt_ppgtt_alloc(void *arg)
148	145	{
149	146	struct drm_i915_private *dev_priv = arg;
150		- struct i915_hw_ppgtt *ppgtt;
	147	+ struct i915_ppgtt *ppgtt;
151	148	u64 size, last, limit;
152	149	int err = 0;
153	150
154	151	/* Allocate a ppggt and try to fill the entire range */
155	152
156		- if (!USES_PPGTT(dev_priv))
	153	+ if (!HAS_PPGTT(dev_priv))
157	154	return 0;
158	155
159		- ppgtt = __hw_ppgtt_create(dev_priv);
	156	+ ppgtt = i915_ppgtt_create(&dev_priv->gt);
160	157	if (IS_ERR(ppgtt))
161	158	return PTR_ERR(ppgtt);
162	159
..	..	@@ -170,59 +167,68 @@
170	167	* This should ensure that we do not run into the oomkiller during
171	168	* the test and take down the machine wilfully.
172	169	*/
173		- limit = totalram_pages << PAGE_SHIFT;
	170	+ limit = totalram_pages() << PAGE_SHIFT;
174	171	limit = min(ppgtt->vm.total, limit);
175	172
176	173	/* Check we can allocate the entire range */
177	174	for (size = 4096; size <= limit; size <<= 2) {
178		- err = ppgtt->vm.allocate_va_range(&ppgtt->vm, 0, size);
	175	+ struct i915_vm_pt_stash stash = {};
	176	+
	177	+ err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size);
	178	+ if (err)
	179	+ goto err_ppgtt_cleanup;
	180	+
	181	+ err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash);
179	182	if (err) {
180		- if (err == -ENOMEM) {
181		- pr_info("[1] Ran out of memory for va_range [0 + %llx] [bit %d]\n",
182		- size, ilog2(size));
183		- err = 0; /* virtual space too large! */
184		- }
	183	+ i915_vm_free_pt_stash(&ppgtt->vm, &stash);
185	184	goto err_ppgtt_cleanup;
186	185	}
187	186
	187	+ ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, size);
188	188	cond_resched();
189	189
190	190	ppgtt->vm.clear_range(&ppgtt->vm, 0, size);
	191	+
	192	+ i915_vm_free_pt_stash(&ppgtt->vm, &stash);
191	193	}
192	194
193	195	/* Check we can incrementally allocate the entire range */
194	196	for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) {
195		- err = ppgtt->vm.allocate_va_range(&ppgtt->vm,
196		- last, size - last);
	197	+ struct i915_vm_pt_stash stash = {};
	198	+
	199	+ err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, size - last);
	200	+ if (err)
	201	+ goto err_ppgtt_cleanup;
	202	+
	203	+ err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash);
197	204	if (err) {
198		- if (err == -ENOMEM) {
199		- pr_info("[2] Ran out of memory for va_range [%llx + %llx] [bit %d]\n",
200		- last, size - last, ilog2(size));
201		- err = 0; /* virtual space too large! */
202		- }
	205	+ i915_vm_free_pt_stash(&ppgtt->vm, &stash);
203	206	goto err_ppgtt_cleanup;
204	207	}
205	208
	209	+ ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash,
	210	+ last, size - last);
206	211	cond_resched();
	212	+
	213	+ i915_vm_free_pt_stash(&ppgtt->vm, &stash);
207	214	}
208	215
209	216	err_ppgtt_cleanup:
210		- mutex_lock(&dev_priv->drm.struct_mutex);
211		- i915_ppgtt_put(ppgtt);
212		- mutex_unlock(&dev_priv->drm.struct_mutex);
	217	+ i915_vm_put(&ppgtt->vm);
213	218	return err;
214	219	}
215	220
216		-static int lowlevel_hole(struct drm_i915_private *i915,
217		- struct i915_address_space *vm,
	221	+static int lowlevel_hole(struct i915_address_space *vm,
218	222	u64 hole_start, u64 hole_end,
219	223	unsigned long end_time)
220	224	{
221	225	I915_RND_STATE(seed_prng);
	226	+ struct i915_vma *mock_vma;
222	227	unsigned int size;
223		- struct i915_vma mock_vma;
224	228
225		- memset(&mock_vma, 0, sizeof(struct i915_vma));
	229	+ mock_vma = kzalloc(sizeof(*mock_vma), GFP_KERNEL);
	230	+ if (!mock_vma)
	231	+ return -ENOMEM;
226	232
227	233	/* Keep creating larger objects until one cannot fit into the hole */
228	234	for (size = 12; (hole_end - hole_start) >> size; size++) {
..	..	@@ -246,8 +252,10 @@
246	252	if (order)
247	253	break;
248	254	} while (count >>= 1);
249		- if (!count)
	255	+ if (!count) {
	256	+ kfree(mock_vma);
250	257	return -ENOMEM;
	258	+ }
251	259	GEM_BUG_ON(!order);
252	260
253	261	GEM_BUG_ON(count * BIT_ULL(size) > vm->total);
..	..	@@ -259,7 +267,7 @@
259	267	* memory. We expect to hit -ENOMEM.
260	268	*/
261	269
262		- obj = fake_dma_object(i915, BIT_ULL(size));
	270	+ obj = fake_dma_object(vm->i915, BIT_ULL(size));
263	271	if (IS_ERR(obj)) {
264	272	kfree(order);
265	273	break;
..	..	@@ -275,6 +283,7 @@
275	283
276	284	for (n = 0; n < count; n++) {
277	285	u64 addr = hole_start + order[n] * BIT_ULL(size);
	286	+ intel_wakeref_t wakeref;
278	287
279	288	GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
280	289
..	..	@@ -285,26 +294,42 @@
285	294	break;
286	295	}
287	296
288		- if (vm->allocate_va_range &&
289		- vm->allocate_va_range(vm, addr, BIT_ULL(size)))
290		- break;
	297	+ if (vm->allocate_va_range) {
	298	+ struct i915_vm_pt_stash stash = {};
291	299
292		- mock_vma.pages = obj->mm.pages;
293		- mock_vma.node.size = BIT_ULL(size);
294		- mock_vma.node.start = addr;
	300	+ if (i915_vm_alloc_pt_stash(vm, &stash,
	301	+ BIT_ULL(size)))
	302	+ break;
295	303
296		- intel_runtime_pm_get(i915);
297		- vm->insert_entries(vm, &mock_vma, I915_CACHE_NONE, 0);
298		- intel_runtime_pm_put(i915);
	304	+ if (i915_vm_pin_pt_stash(vm, &stash)) {
	305	+ i915_vm_free_pt_stash(vm, &stash);
	306	+ break;
	307	+ }
	308	+
	309	+ vm->allocate_va_range(vm, &stash,
	310	+ addr, BIT_ULL(size));
	311	+
	312	+ i915_vm_free_pt_stash(vm, &stash);
	313	+ }
	314	+
	315	+ mock_vma->pages = obj->mm.pages;
	316	+ mock_vma->node.size = BIT_ULL(size);
	317	+ mock_vma->node.start = addr;
	318	+
	319	+ with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref)
	320	+ vm->insert_entries(vm, mock_vma,
	321	+ I915_CACHE_NONE, 0);
299	322	}
300	323	count = n;
301	324
302	325	i915_random_reorder(order, count, &prng);
303	326	for (n = 0; n < count; n++) {
304	327	u64 addr = hole_start + order[n] * BIT_ULL(size);
	328	+ intel_wakeref_t wakeref;
305	329
306	330	GEM_BUG_ON(addr + BIT_ULL(size) > vm->total);
307		- vm->clear_range(vm, addr, BIT_ULL(size));
	331	+ with_intel_runtime_pm(vm->gt->uncore->rpm, wakeref)
	332	+ vm->clear_range(vm, addr, BIT_ULL(size));
308	333	}
309	334
310	335	i915_gem_object_unpin_pages(obj);
..	..	@@ -312,9 +337,10 @@
312	337
313	338	kfree(order);
314	339
315		- cleanup_freed_objects(i915);
	340	+ cleanup_freed_objects(vm->i915);
316	341	}
317	342
	343	+ kfree(mock_vma);
318	344	return 0;
319	345	}
320	346
..	..	@@ -330,17 +356,13 @@
330	356	vma = i915_vma_instance(obj, vm, NULL);
331	357	if (!IS_ERR(vma))
332	358	ignored = i915_vma_unbind(vma);
333		- /* Only ppgtt vma may be closed before the object is freed */
334		- if (!IS_ERR(vma) && !i915_vma_is_ggtt(vma))
335		- i915_vma_close(vma);
336	359
337	360	list_del(&obj->st_link);
338	361	i915_gem_object_put(obj);
339	362	}
340	363	}
341	364
342		-static int fill_hole(struct drm_i915_private *i915,
343		- struct i915_address_space *vm,
	365	+static int fill_hole(struct i915_address_space *vm,
344	366	u64 hole_start, u64 hole_end,
345	367	unsigned long end_time)
346	368	{
..	..	@@ -373,7 +395,7 @@
373	395	{ }
374	396	}, *p;
375	397
376		- obj = fake_dma_object(i915, full_size);
	398	+ obj = fake_dma_object(vm->i915, full_size);
377	399	if (IS_ERR(obj))
378	400	break;
379	401
..	..	@@ -541,7 +563,7 @@
541	563	}
542	564
543	565	close_object_list(&objects, vm);
544		- cleanup_freed_objects(i915);
	566	+ cleanup_freed_objects(vm->i915);
545	567	}
546	568
547	569	return 0;
..	..	@@ -551,8 +573,7 @@
551	573	return err;
552	574	}
553	575
554		-static int walk_hole(struct drm_i915_private *i915,
555		- struct i915_address_space *vm,
	576	+static int walk_hole(struct i915_address_space *vm,
556	577	u64 hole_start, u64 hole_end,
557	578	unsigned long end_time)
558	579	{
..	..	@@ -574,7 +595,7 @@
574	595	u64 addr;
575	596	int err = 0;
576	597
577		- obj = fake_dma_object(i915, size << PAGE_SHIFT);
	598	+ obj = fake_dma_object(vm->i915, size << PAGE_SHIFT);
578	599	if (IS_ERR(obj))
579	600	break;
580	601
..	..	@@ -592,7 +613,7 @@
592	613	pr_err("%s bind failed at %llx + %llx [hole %llx- %llx] with err=%d\n",
593	614	__func__, addr, vma->size,
594	615	hole_start, hole_end, err);
595		- goto err_close;
	616	+ goto err_put;
596	617	}
597	618	i915_vma_unpin(vma);
598	619
..	..	@@ -601,14 +622,14 @@
601	622	pr_err("%s incorrect at %llx + %llx\n",
602	623	__func__, addr, vma->size);
603	624	err = -EINVAL;
604		- goto err_close;
	625	+ goto err_put;
605	626	}
606	627
607	628	err = i915_vma_unbind(vma);
608	629	if (err) {
609	630	pr_err("%s unbind failed at %llx + %llx with err=%d\n",
610	631	__func__, addr, vma->size, err);
611		- goto err_close;
	632	+ goto err_put;
612	633	}
613	634
614	635	GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
..	..	@@ -617,26 +638,22 @@
617	638	"%s timed out at %llx\n",
618	639	__func__, addr)) {
619	640	err = -EINTR;
620		- goto err_close;
	641	+ goto err_put;
621	642	}
622	643	}
623	644
624		-err_close:
625		- if (!i915_vma_is_ggtt(vma))
626		- i915_vma_close(vma);
627	645	err_put:
628	646	i915_gem_object_put(obj);
629	647	if (err)
630	648	return err;
631	649
632		- cleanup_freed_objects(i915);
	650	+ cleanup_freed_objects(vm->i915);
633	651	}
634	652
635	653	return 0;
636	654	}
637	655
638		-static int pot_hole(struct drm_i915_private *i915,
639		- struct i915_address_space *vm,
	656	+static int pot_hole(struct i915_address_space *vm,
640	657	u64 hole_start, u64 hole_end,
641	658	unsigned long end_time)
642	659	{
..	..	@@ -650,7 +667,7 @@
650	667	if (i915_is_ggtt(vm))
651	668	flags \|= PIN_GLOBAL;
652	669
653		- obj = i915_gem_object_create_internal(i915, 2 * I915_GTT_PAGE_SIZE);
	670	+ obj = i915_gem_object_create_internal(vm->i915, 2 * I915_GTT_PAGE_SIZE);
654	671	if (IS_ERR(obj))
655	672	return PTR_ERR(obj);
656	673
..	..	@@ -677,7 +694,7 @@
677	694	addr,
678	695	hole_start, hole_end,
679	696	err);
680		- goto err;
	697	+ goto err_obj;
681	698	}
682	699
683	700	if (!drm_mm_node_allocated(&vma->node) \|\|
..	..	@@ -687,7 +704,7 @@
687	704	i915_vma_unpin(vma);
688	705	err = i915_vma_unbind(vma);
689	706	err = -EINVAL;
690		- goto err;
	707	+ goto err_obj;
691	708	}
692	709
693	710	i915_vma_unpin(vma);
..	..	@@ -699,20 +716,16 @@
699	716	"%s timed out after %d/%d\n",
700	717	__func__, pot, fls64(hole_end - 1) - 1)) {
701	718	err = -EINTR;
702		- goto err;
	719	+ goto err_obj;
703	720	}
704	721	}
705	722
706		-err:
707		- if (!i915_vma_is_ggtt(vma))
708		- i915_vma_close(vma);
709	723	err_obj:
710	724	i915_gem_object_put(obj);
711	725	return err;
712	726	}
713	727
714		-static int drunk_hole(struct drm_i915_private *i915,
715		- struct i915_address_space *vm,
	728	+static int drunk_hole(struct i915_address_space *vm,
716	729	u64 hole_start, u64 hole_end,
717	730	unsigned long end_time)
718	731	{
..	..	@@ -757,7 +770,7 @@
757	770	* memory. We expect to hit -ENOMEM.
758	771	*/
759	772
760		- obj = fake_dma_object(i915, BIT_ULL(size));
	773	+ obj = fake_dma_object(vm->i915, BIT_ULL(size));
761	774	if (IS_ERR(obj)) {
762	775	kfree(order);
763	776	break;
..	..	@@ -781,7 +794,7 @@
781	794	addr, BIT_ULL(size),
782	795	hole_start, hole_end,
783	796	err);
784		- goto err;
	797	+ goto err_obj;
785	798	}
786	799
787	800	if (!drm_mm_node_allocated(&vma->node) \|\|
..	..	@@ -791,7 +804,7 @@
791	804	i915_vma_unpin(vma);
792	805	err = i915_vma_unbind(vma);
793	806	err = -EINVAL;
794		- goto err;
	807	+ goto err_obj;
795	808	}
796	809
797	810	i915_vma_unpin(vma);
..	..	@@ -802,27 +815,23 @@
802	815	"%s timed out after %d/%d\n",
803	816	__func__, n, count)) {
804	817	err = -EINTR;
805		- goto err;
	818	+ goto err_obj;
806	819	}
807	820	}
808	821
809		-err:
810		- if (!i915_vma_is_ggtt(vma))
811		- i915_vma_close(vma);
812	822	err_obj:
813	823	i915_gem_object_put(obj);
814	824	kfree(order);
815	825	if (err)
816	826	return err;
817	827
818		- cleanup_freed_objects(i915);
	828	+ cleanup_freed_objects(vm->i915);
819	829	}
820	830
821	831	return 0;
822	832	}
823	833
824		-static int __shrink_hole(struct drm_i915_private *i915,
825		- struct i915_address_space *vm,
	834	+static int __shrink_hole(struct i915_address_space *vm,
826	835	u64 hole_start, u64 hole_end,
827	836	unsigned long end_time)
828	837	{
..	..	@@ -839,7 +848,7 @@
839	848	u64 size = BIT_ULL(order++);
840	849
841	850	size = min(size, hole_end - addr);
842		- obj = fake_dma_object(i915, size);
	851	+ obj = fake_dma_object(vm->i915, size);
843	852	if (IS_ERR(obj)) {
844	853	err = PTR_ERR(obj);
845	854	break;
..	..	@@ -875,6 +884,15 @@
875	884	i915_vma_unpin(vma);
876	885	addr += size;
877	886
	887	+ /*
	888	+ * Since we are injecting allocation faults at random intervals,
	889	+ * wait for this allocation to complete before we change the
	890	+ * faultinjection.
	891	+ */
	892	+ err = i915_vma_sync(vma);
	893	+ if (err)
	894	+ break;
	895	+
878	896	if (igt_timeout(end_time,
879	897	"%s timed out at ofset %llx [%llx - %llx]\n",
880	898	__func__, addr, hole_start, hole_end)) {
..	..	@@ -884,12 +902,11 @@
884	902	}
885	903
886	904	close_object_list(&objects, vm);
887		- cleanup_freed_objects(i915);
	905	+ cleanup_freed_objects(vm->i915);
888	906	return err;
889	907	}
890	908
891		-static int shrink_hole(struct drm_i915_private *i915,
892		- struct i915_address_space *vm,
	909	+static int shrink_hole(struct i915_address_space *vm,
893	910	u64 hole_start, u64 hole_end,
894	911	unsigned long end_time)
895	912	{
..	..	@@ -901,7 +918,7 @@
901	918
902	919	for_each_prime_number_from(prime, 0, ULONG_MAX - 1) {
903	920	vm->fault_attr.interval = prime;
904		- err = __shrink_hole(i915, vm, hole_start, hole_end, end_time);
	921	+ err = __shrink_hole(vm, hole_start, hole_end, end_time);
905	922	if (err)
906	923	break;
907	924	}
..	..	@@ -911,8 +928,7 @@
911	928	return err;
912	929	}
913	930
914		-static int shrink_boom(struct drm_i915_private *i915,
915		- struct i915_address_space *vm,
	931	+static int shrink_boom(struct i915_address_space *vm,
916	932	u64 hole_start, u64 hole_end,
917	933	unsigned long end_time)
918	934	{
..	..	@@ -934,7 +950,7 @@
934	950	unsigned int size = sizes[i];
935	951	struct i915_vma *vma;
936	952
937		- purge = fake_dma_object(i915, size);
	953	+ purge = fake_dma_object(vm->i915, size);
938	954	if (IS_ERR(purge))
939	955	return PTR_ERR(purge);
940	956
..	..	@@ -951,7 +967,7 @@
951	967	/* Should now be ripe for purging */
952	968	i915_vma_unpin(vma);
953	969
954		- explode = fake_dma_object(i915, size);
	970	+ explode = fake_dma_object(vm->i915, size);
955	971	if (IS_ERR(explode)) {
956	972	err = PTR_ERR(explode);
957	973	goto err_purge;
..	..	@@ -977,7 +993,7 @@
977	993	i915_gem_object_put(explode);
978	994
979	995	memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
980		- cleanup_freed_objects(i915);
	996	+ cleanup_freed_objects(vm->i915);
981	997	}
982	998
983	999	return 0;
..	..	@@ -991,40 +1007,36 @@
991	1007	}
992	1008
993	1009	static int exercise_ppgtt(struct drm_i915_private *dev_priv,
994		- int (func)(struct drm_i915_private i915,
995		- struct i915_address_space *vm,
	1010	+ int (func)(struct i915_address_space vm,
996	1011	u64 hole_start, u64 hole_end,
997	1012	unsigned long end_time))
998	1013	{
999		- struct drm_file *file;
1000		- struct i915_hw_ppgtt *ppgtt;
	1014	+ struct i915_ppgtt *ppgtt;
1001	1015	IGT_TIMEOUT(end_time);
	1016	+ struct file *file;
1002	1017	int err;
1003	1018
1004		- if (!USES_FULL_PPGTT(dev_priv))
	1019	+ if (!HAS_FULL_PPGTT(dev_priv))
1005	1020	return 0;
1006	1021
1007	1022	file = mock_file(dev_priv);
1008	1023	if (IS_ERR(file))
1009	1024	return PTR_ERR(file);
1010	1025
1011		- mutex_lock(&dev_priv->drm.struct_mutex);
1012		- ppgtt = i915_ppgtt_create(dev_priv, file->driver_priv);
	1026	+ ppgtt = i915_ppgtt_create(&dev_priv->gt);
1013	1027	if (IS_ERR(ppgtt)) {
1014	1028	err = PTR_ERR(ppgtt);
1015		- goto out_unlock;
	1029	+ goto out_free;
1016	1030	}
1017	1031	GEM_BUG_ON(offset_in_page(ppgtt->vm.total));
1018		- GEM_BUG_ON(ppgtt->vm.closed);
	1032	+ GEM_BUG_ON(!atomic_read(&ppgtt->vm.open));
1019	1033
1020		- err = func(dev_priv, &ppgtt->vm, 0, ppgtt->vm.total, end_time);
	1034	+ err = func(&ppgtt->vm, 0, ppgtt->vm.total, end_time);
1021	1035
1022		- i915_ppgtt_close(&ppgtt->vm);
1023		- i915_ppgtt_put(ppgtt);
1024		-out_unlock:
1025		- mutex_unlock(&dev_priv->drm.struct_mutex);
	1036	+ i915_vm_put(&ppgtt->vm);
1026	1037
1027		- mock_file_free(dev_priv, file);
	1038	+out_free:
	1039	+ fput(file);
1028	1040	return err;
1029	1041	}
1030	1042
..	..	@@ -1075,8 +1087,7 @@
1075	1087	}
1076	1088
1077	1089	static int exercise_ggtt(struct drm_i915_private *i915,
1078		- int (func)(struct drm_i915_private i915,
1079		- struct i915_address_space *vm,
	1090	+ int (func)(struct i915_address_space vm,
1080	1091	u64 hole_start, u64 hole_end,
1081	1092	unsigned long end_time))
1082	1093	{
..	..	@@ -1086,7 +1097,6 @@
1086	1097	IGT_TIMEOUT(end_time);
1087	1098	int err = 0;
1088	1099
1089		- mutex_lock(&i915->drm.struct_mutex);
1090	1100	restart:
1091	1101	list_sort(NULL, &ggtt->vm.mm.hole_stack, sort_holes);
1092	1102	drm_mm_for_each_hole(node, &ggtt->vm.mm, hole_start, hole_end) {
..	..	@@ -1099,7 +1109,7 @@
1099	1109	if (hole_start >= hole_end)
1100	1110	continue;
1101	1111
1102		- err = func(i915, &ggtt->vm, hole_start, hole_end, end_time);
	1112	+ err = func(&ggtt->vm, hole_start, hole_end, end_time);
1103	1113	if (err)
1104	1114	break;
1105	1115
..	..	@@ -1107,7 +1117,6 @@
1107	1117	last = hole_end;
1108	1118	goto restart;
1109	1119	}
1110		- mutex_unlock(&i915->drm.struct_mutex);
1111	1120
1112	1121	return err;
1113	1122	}
..	..	@@ -1144,32 +1153,34 @@
1144	1153	struct drm_i915_private *i915 = arg;
1145	1154	struct i915_ggtt *ggtt = &i915->ggtt;
1146	1155	struct drm_i915_gem_object *obj;
	1156	+ intel_wakeref_t wakeref;
1147	1157	struct drm_mm_node tmp;
1148	1158	unsigned int *order, n;
1149	1159	int err;
1150	1160
1151		- mutex_lock(&i915->drm.struct_mutex);
	1161	+ if (!i915_ggtt_has_aperture(ggtt))
	1162	+ return 0;
1152	1163
1153	1164	obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1154		- if (IS_ERR(obj)) {
1155		- err = PTR_ERR(obj);
1156		- goto out_unlock;
1157		- }
	1165	+ if (IS_ERR(obj))
	1166	+ return PTR_ERR(obj);
1158	1167
1159	1168	err = i915_gem_object_pin_pages(obj);
1160	1169	if (err)
1161	1170	goto out_free;
1162	1171
1163	1172	memset(&tmp, 0, sizeof(tmp));
	1173	+ mutex_lock(&ggtt->vm.mutex);
1164	1174	err = drm_mm_insert_node_in_range(&ggtt->vm.mm, &tmp,
1165	1175	count * PAGE_SIZE, 0,
1166	1176	I915_COLOR_UNEVICTABLE,
1167	1177	0, ggtt->mappable_end,
1168	1178	DRM_MM_INSERT_LOW);
	1179	+ mutex_unlock(&ggtt->vm.mutex);
1169	1180	if (err)
1170	1181	goto out_unpin;
1171	1182
1172		- intel_runtime_pm_get(i915);
	1183	+ wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1173	1184
1174	1185	for (n = 0; n < count; n++) {
1175	1186	u64 offset = tmp.start + n * PAGE_SIZE;
..	..	@@ -1193,7 +1204,7 @@
1193	1204	iowrite32(n, vaddr + n);
1194	1205	io_mapping_unmap_atomic(vaddr);
1195	1206	}
1196		- i915_gem_flush_ggtt_writes(i915);
	1207	+ intel_gt_flush_ggtt_writes(ggtt->vm.gt);
1197	1208
1198	1209	i915_random_reorder(order, count, &prng);
1199	1210	for (n = 0; n < count; n++) {
..	..	@@ -1216,14 +1227,14 @@
1216	1227	kfree(order);
1217	1228	out_remove:
1218	1229	ggtt->vm.clear_range(&ggtt->vm, tmp.start, tmp.size);
1219		- intel_runtime_pm_put(i915);
	1230	+ intel_runtime_pm_put(&i915->runtime_pm, wakeref);
	1231	+ mutex_lock(&ggtt->vm.mutex);
1220	1232	drm_mm_remove_node(&tmp);
	1233	+ mutex_unlock(&ggtt->vm.mutex);
1221	1234	out_unpin:
1222	1235	i915_gem_object_unpin_pages(obj);
1223	1236	out_free:
1224	1237	i915_gem_object_put(obj);
1225		-out_unlock:
1226		- mutex_unlock(&i915->drm.struct_mutex);
1227	1238	return err;
1228	1239	}
1229	1240
..	..	@@ -1231,22 +1242,26 @@
1231	1242	{
1232	1243	struct drm_i915_gem_object *obj = vma->obj;
1233	1244
1234		- obj->bind_count++; /* track for eviction later */
1235	1245	__i915_gem_object_pin_pages(obj);
1236	1246
	1247	+ GEM_BUG_ON(vma->pages);
	1248	+ atomic_set(&vma->pages_count, I915_VMA_PAGES_ACTIVE);
	1249	+ __i915_gem_object_pin_pages(obj);
1237	1250	vma->pages = obj->mm.pages;
1238		- list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
	1251	+
	1252	+ mutex_lock(&vma->vm->mutex);
	1253	+ list_add_tail(&vma->vm_link, &vma->vm->bound_list);
	1254	+ mutex_unlock(&vma->vm->mutex);
1239	1255	}
1240	1256
1241	1257	static int exercise_mock(struct drm_i915_private *i915,
1242		- int (func)(struct drm_i915_private i915,
1243		- struct i915_address_space *vm,
	1258	+ int (func)(struct i915_address_space vm,
1244	1259	u64 hole_start, u64 hole_end,
1245	1260	unsigned long end_time))
1246	1261	{
1247		- const u64 limit = totalram_pages << PAGE_SHIFT;
	1262	+ const u64 limit = totalram_pages() << PAGE_SHIFT;
	1263	+ struct i915_address_space *vm;
1248	1264	struct i915_gem_context *ctx;
1249		- struct i915_hw_ppgtt *ppgtt;
1250	1265	IGT_TIMEOUT(end_time);
1251	1266	int err;
1252	1267
..	..	@@ -1254,10 +1269,9 @@
1254	1269	if (!ctx)
1255	1270	return -ENOMEM;
1256	1271
1257		- ppgtt = ctx->ppgtt;
1258		- GEM_BUG_ON(!ppgtt);
1259		-
1260		- err = func(i915, &ppgtt->vm, 0, min(ppgtt->vm.total, limit), end_time);
	1272	+ vm = i915_gem_context_get_vm_rcu(ctx);
	1273	+ err = func(vm, 0, min(vm->total, limit), end_time);
	1274	+ i915_vm_put(vm);
1261	1275
1262	1276	mock_context_close(ctx);
1263	1277	return err;
..	..	@@ -1265,28 +1279,37 @@
1265	1279
1266	1280	static int igt_mock_fill(void *arg)
1267	1281	{
1268		- return exercise_mock(arg, fill_hole);
	1282	+ struct i915_ggtt *ggtt = arg;
	1283	+
	1284	+ return exercise_mock(ggtt->vm.i915, fill_hole);
1269	1285	}
1270	1286
1271	1287	static int igt_mock_walk(void *arg)
1272	1288	{
1273		- return exercise_mock(arg, walk_hole);
	1289	+ struct i915_ggtt *ggtt = arg;
	1290	+
	1291	+ return exercise_mock(ggtt->vm.i915, walk_hole);
1274	1292	}
1275	1293
1276	1294	static int igt_mock_pot(void *arg)
1277	1295	{
1278		- return exercise_mock(arg, pot_hole);
	1296	+ struct i915_ggtt *ggtt = arg;
	1297	+
	1298	+ return exercise_mock(ggtt->vm.i915, pot_hole);
1279	1299	}
1280	1300
1281	1301	static int igt_mock_drunk(void *arg)
1282	1302	{
1283		- return exercise_mock(arg, drunk_hole);
	1303	+ struct i915_ggtt *ggtt = arg;
	1304	+
	1305	+ return exercise_mock(ggtt->vm.i915, drunk_hole);
1284	1306	}
1285	1307
1286	1308	static int igt_gtt_reserve(void *arg)
1287	1309	{
1288		- struct drm_i915_private *i915 = arg;
	1310	+ struct i915_ggtt *ggtt = arg;
1289	1311	struct drm_i915_gem_object obj, on;
	1312	+ I915_RND_STATE(prng);
1290	1313	LIST_HEAD(objects);
1291	1314	u64 total;
1292	1315	int err = -ENODEV;
..	..	@@ -1298,11 +1321,12 @@
1298	1321
1299	1322	/* Start by filling the GGTT */
1300	1323	for (total = 0;
1301		- total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.vm.total;
1302		- total += 2*I915_GTT_PAGE_SIZE) {
	1324	+ total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
	1325	+ total += 2 * I915_GTT_PAGE_SIZE) {
1303	1326	struct i915_vma *vma;
1304	1327
1305		- obj = i915_gem_object_create_internal(i915, 2*PAGE_SIZE);
	1328	+ obj = i915_gem_object_create_internal(ggtt->vm.i915,
	1329	+ 2 * PAGE_SIZE);
1306	1330	if (IS_ERR(obj)) {
1307	1331	err = PTR_ERR(obj);
1308	1332	goto out;
..	..	@@ -1316,20 +1340,22 @@
1316	1340
1317	1341	list_add(&obj->st_link, &objects);
1318	1342
1319		- vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
	1343	+ vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1320	1344	if (IS_ERR(vma)) {
1321	1345	err = PTR_ERR(vma);
1322	1346	goto out;
1323	1347	}
1324	1348
1325		- err = i915_gem_gtt_reserve(&i915->ggtt.vm, &vma->node,
	1349	+ mutex_lock(&ggtt->vm.mutex);
	1350	+ err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
1326	1351	obj->base.size,
1327	1352	total,
1328	1353	obj->cache_level,
1329	1354	0);
	1355	+ mutex_unlock(&ggtt->vm.mutex);
1330	1356	if (err) {
1331	1357	pr_err("i915_gem_gtt_reserve (pass 1) failed at %llu/%llu with err=%d\n",
1332		- total, i915->ggtt.vm.total, err);
	1358	+ total, ggtt->vm.total, err);
1333	1359	goto out;
1334	1360	}
1335	1361	track_vma_bind(vma);
..	..	@@ -1347,11 +1373,12 @@
1347	1373
1348	1374	/* Now we start forcing evictions */
1349	1375	for (total = I915_GTT_PAGE_SIZE;
1350		- total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.vm.total;
1351		- total += 2*I915_GTT_PAGE_SIZE) {
	1376	+ total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
	1377	+ total += 2 * I915_GTT_PAGE_SIZE) {
1352	1378	struct i915_vma *vma;
1353	1379
1354		- obj = i915_gem_object_create_internal(i915, 2*PAGE_SIZE);
	1380	+ obj = i915_gem_object_create_internal(ggtt->vm.i915,
	1381	+ 2 * PAGE_SIZE);
1355	1382	if (IS_ERR(obj)) {
1356	1383	err = PTR_ERR(obj);
1357	1384	goto out;
..	..	@@ -1365,20 +1392,22 @@
1365	1392
1366	1393	list_add(&obj->st_link, &objects);
1367	1394
1368		- vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
	1395	+ vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1369	1396	if (IS_ERR(vma)) {
1370	1397	err = PTR_ERR(vma);
1371	1398	goto out;
1372	1399	}
1373	1400
1374		- err = i915_gem_gtt_reserve(&i915->ggtt.vm, &vma->node,
	1401	+ mutex_lock(&ggtt->vm.mutex);
	1402	+ err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
1375	1403	obj->base.size,
1376	1404	total,
1377	1405	obj->cache_level,
1378	1406	0);
	1407	+ mutex_unlock(&ggtt->vm.mutex);
1379	1408	if (err) {
1380	1409	pr_err("i915_gem_gtt_reserve (pass 2) failed at %llu/%llu with err=%d\n",
1381		- total, i915->ggtt.vm.total, err);
	1410	+ total, ggtt->vm.total, err);
1382	1411	goto out;
1383	1412	}
1384	1413	track_vma_bind(vma);
..	..	@@ -1399,7 +1428,7 @@
1399	1428	struct i915_vma *vma;
1400	1429	u64 offset;
1401	1430
1402		- vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
	1431	+ vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1403	1432	if (IS_ERR(vma)) {
1404	1433	err = PTR_ERR(vma);
1405	1434	goto out;
..	..	@@ -1411,18 +1440,21 @@
1411	1440	goto out;
1412	1441	}
1413	1442
1414		- offset = random_offset(0, i915->ggtt.vm.total,
1415		- 2*I915_GTT_PAGE_SIZE,
1416		- I915_GTT_MIN_ALIGNMENT);
	1443	+ offset = igt_random_offset(&prng,
	1444	+ 0, ggtt->vm.total,
	1445	+ 2 * I915_GTT_PAGE_SIZE,
	1446	+ I915_GTT_MIN_ALIGNMENT);
1417	1447
1418		- err = i915_gem_gtt_reserve(&i915->ggtt.vm, &vma->node,
	1448	+ mutex_lock(&ggtt->vm.mutex);
	1449	+ err = i915_gem_gtt_reserve(&ggtt->vm, &vma->node,
1419	1450	obj->base.size,
1420	1451	offset,
1421	1452	obj->cache_level,
1422	1453	0);
	1454	+ mutex_unlock(&ggtt->vm.mutex);
1423	1455	if (err) {
1424	1456	pr_err("i915_gem_gtt_reserve (pass 3) failed at %llu/%llu with err=%d\n",
1425		- total, i915->ggtt.vm.total, err);
	1457	+ total, ggtt->vm.total, err);
1426	1458	goto out;
1427	1459	}
1428	1460	track_vma_bind(vma);
..	..	@@ -1448,7 +1480,7 @@
1448	1480
1449	1481	static int igt_gtt_insert(void *arg)
1450	1482	{
1451		- struct drm_i915_private *i915 = arg;
	1483	+ struct i915_ggtt *ggtt = arg;
1452	1484	struct drm_i915_gem_object obj, on;
1453	1485	struct drm_mm_node tmp = {};
1454	1486	const struct invalid_insert {
..	..	@@ -1457,8 +1489,8 @@
1457	1489	u64 start, end;
1458	1490	} invalid_insert[] = {
1459	1491	{
1460		- i915->ggtt.vm.total + I915_GTT_PAGE_SIZE, 0,
1461		- 0, i915->ggtt.vm.total,
	1492	+ ggtt->vm.total + I915_GTT_PAGE_SIZE, 0,
	1493	+ 0, ggtt->vm.total,
1462	1494	},
1463	1495	{
1464	1496	2*I915_GTT_PAGE_SIZE, 0,
..	..	@@ -1488,11 +1520,13 @@
1488	1520
1489	1521	/* Check a couple of obviously invalid requests */
1490	1522	for (ii = invalid_insert; ii->size; ii++) {
1491		- err = i915_gem_gtt_insert(&i915->ggtt.vm, &tmp,
	1523	+ mutex_lock(&ggtt->vm.mutex);
	1524	+ err = i915_gem_gtt_insert(&ggtt->vm, &tmp,
1492	1525	ii->size, ii->alignment,
1493	1526	I915_COLOR_UNEVICTABLE,
1494	1527	ii->start, ii->end,
1495	1528	0);
	1529	+ mutex_unlock(&ggtt->vm.mutex);
1496	1530	if (err != -ENOSPC) {
1497	1531	pr_err("Invalid i915_gem_gtt_insert(.size=%llx, .alignment=%llx, .start=%llx, .end=%llx) succeeded (err=%d)\n",
1498	1532	ii->size, ii->alignment, ii->start, ii->end,
..	..	@@ -1503,11 +1537,12 @@
1503	1537
1504	1538	/* Start by filling the GGTT */
1505	1539	for (total = 0;
1506		- total + I915_GTT_PAGE_SIZE <= i915->ggtt.vm.total;
	1540	+ total + I915_GTT_PAGE_SIZE <= ggtt->vm.total;
1507	1541	total += I915_GTT_PAGE_SIZE) {
1508	1542	struct i915_vma *vma;
1509	1543
1510		- obj = i915_gem_object_create_internal(i915, I915_GTT_PAGE_SIZE);
	1544	+ obj = i915_gem_object_create_internal(ggtt->vm.i915,
	1545	+ I915_GTT_PAGE_SIZE);
1511	1546	if (IS_ERR(obj)) {
1512	1547	err = PTR_ERR(obj);
1513	1548	goto out;
..	..	@@ -1521,16 +1556,18 @@
1521	1556
1522	1557	list_add(&obj->st_link, &objects);
1523	1558
1524		- vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
	1559	+ vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1525	1560	if (IS_ERR(vma)) {
1526	1561	err = PTR_ERR(vma);
1527	1562	goto out;
1528	1563	}
1529	1564
1530		- err = i915_gem_gtt_insert(&i915->ggtt.vm, &vma->node,
	1565	+ mutex_lock(&ggtt->vm.mutex);
	1566	+ err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
1531	1567	obj->base.size, 0, obj->cache_level,
1532		- 0, i915->ggtt.vm.total,
	1568	+ 0, ggtt->vm.total,
1533	1569	0);
	1570	+ mutex_unlock(&ggtt->vm.mutex);
1534	1571	if (err == -ENOSPC) {
1535	1572	/* maxed out the GGTT space */
1536	1573	i915_gem_object_put(obj);
..	..	@@ -1538,7 +1575,7 @@
1538	1575	}
1539	1576	if (err) {
1540	1577	pr_err("i915_gem_gtt_insert (pass 1) failed at %llu/%llu with err=%d\n",
1541		- total, i915->ggtt.vm.total, err);
	1578	+ total, ggtt->vm.total, err);
1542	1579	goto out;
1543	1580	}
1544	1581	track_vma_bind(vma);
..	..	@@ -1550,7 +1587,7 @@
1550	1587	list_for_each_entry(obj, &objects, st_link) {
1551	1588	struct i915_vma *vma;
1552	1589
1553		- vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
	1590	+ vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1554	1591	if (IS_ERR(vma)) {
1555	1592	err = PTR_ERR(vma);
1556	1593	goto out;
..	..	@@ -1570,7 +1607,7 @@
1570	1607	struct i915_vma *vma;
1571	1608	u64 offset;
1572	1609
1573		- vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
	1610	+ vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1574	1611	if (IS_ERR(vma)) {
1575	1612	err = PTR_ERR(vma);
1576	1613	goto out;
..	..	@@ -1585,13 +1622,15 @@
1585	1622	goto out;
1586	1623	}
1587	1624
1588		- err = i915_gem_gtt_insert(&i915->ggtt.vm, &vma->node,
	1625	+ mutex_lock(&ggtt->vm.mutex);
	1626	+ err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
1589	1627	obj->base.size, 0, obj->cache_level,
1590		- 0, i915->ggtt.vm.total,
	1628	+ 0, ggtt->vm.total,
1591	1629	0);
	1630	+ mutex_unlock(&ggtt->vm.mutex);
1592	1631	if (err) {
1593	1632	pr_err("i915_gem_gtt_insert (pass 2) failed at %llu/%llu with err=%d\n",
1594		- total, i915->ggtt.vm.total, err);
	1633	+ total, ggtt->vm.total, err);
1595	1634	goto out;
1596	1635	}
1597	1636	track_vma_bind(vma);
..	..	@@ -1607,11 +1646,12 @@
1607	1646
1608	1647	/* And then force evictions */
1609	1648	for (total = 0;
1610		- total + 2*I915_GTT_PAGE_SIZE <= i915->ggtt.vm.total;
1611		- total += 2*I915_GTT_PAGE_SIZE) {
	1649	+ total + 2 * I915_GTT_PAGE_SIZE <= ggtt->vm.total;
	1650	+ total += 2 * I915_GTT_PAGE_SIZE) {
1612	1651	struct i915_vma *vma;
1613	1652
1614		- obj = i915_gem_object_create_internal(i915, 2*I915_GTT_PAGE_SIZE);
	1653	+ obj = i915_gem_object_create_internal(ggtt->vm.i915,
	1654	+ 2 * I915_GTT_PAGE_SIZE);
1615	1655	if (IS_ERR(obj)) {
1616	1656	err = PTR_ERR(obj);
1617	1657	goto out;
..	..	@@ -1625,19 +1665,21 @@
1625	1665
1626	1666	list_add(&obj->st_link, &objects);
1627	1667
1628		- vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
	1668	+ vma = i915_vma_instance(obj, &ggtt->vm, NULL);
1629	1669	if (IS_ERR(vma)) {
1630	1670	err = PTR_ERR(vma);
1631	1671	goto out;
1632	1672	}
1633	1673
1634		- err = i915_gem_gtt_insert(&i915->ggtt.vm, &vma->node,
	1674	+ mutex_lock(&ggtt->vm.mutex);
	1675	+ err = i915_gem_gtt_insert(&ggtt->vm, &vma->node,
1635	1676	obj->base.size, 0, obj->cache_level,
1636		- 0, i915->ggtt.vm.total,
	1677	+ 0, ggtt->vm.total,
1637	1678	0);
	1679	+ mutex_unlock(&ggtt->vm.mutex);
1638	1680	if (err) {
1639	1681	pr_err("i915_gem_gtt_insert (pass 3) failed at %llu/%llu with err=%d\n",
1640		- total, i915->ggtt.vm.total, err);
	1682	+ total, ggtt->vm.total, err);
1641	1683	goto out;
1642	1684	}
1643	1685	track_vma_bind(vma);
..	..	@@ -1664,17 +1706,345 @@
1664	1706	SUBTEST(igt_gtt_insert),
1665	1707	};
1666	1708	struct drm_i915_private *i915;
	1709	+ struct i915_ggtt *ggtt;
1667	1710	int err;
1668	1711
1669	1712	i915 = mock_gem_device();
1670	1713	if (!i915)
1671	1714	return -ENOMEM;
1672	1715
1673		- mutex_lock(&i915->drm.struct_mutex);
1674		- err = i915_subtests(tests, i915);
1675		- mutex_unlock(&i915->drm.struct_mutex);
	1716	+ ggtt = kmalloc(sizeof(*ggtt), GFP_KERNEL);
	1717	+ if (!ggtt) {
	1718	+ err = -ENOMEM;
	1719	+ goto out_put;
	1720	+ }
	1721	+ mock_init_ggtt(i915, ggtt);
1676	1722
1677		- drm_dev_put(&i915->drm);
	1723	+ err = i915_subtests(tests, ggtt);
	1724	+
	1725	+ mock_device_flush(i915);
	1726	+ i915_gem_drain_freed_objects(i915);
	1727	+ mock_fini_ggtt(ggtt);
	1728	+ kfree(ggtt);
	1729	+out_put:
	1730	+ mock_destroy_device(i915);
	1731	+ return err;
	1732	+}
	1733	+
	1734	+static int context_sync(struct intel_context *ce)
	1735	+{
	1736	+ struct i915_request *rq;
	1737	+ long timeout;
	1738	+
	1739	+ rq = intel_context_create_request(ce);
	1740	+ if (IS_ERR(rq))
	1741	+ return PTR_ERR(rq);
	1742	+
	1743	+ i915_request_get(rq);
	1744	+ i915_request_add(rq);
	1745	+
	1746	+ timeout = i915_request_wait(rq, 0, HZ / 5);
	1747	+ i915_request_put(rq);
	1748	+
	1749	+ return timeout < 0 ? -EIO : 0;
	1750	+}
	1751	+
	1752	+static struct i915_request *
	1753	+submit_batch(struct intel_context *ce, u64 addr)
	1754	+{
	1755	+ struct i915_request *rq;
	1756	+ int err;
	1757	+
	1758	+ rq = intel_context_create_request(ce);
	1759	+ if (IS_ERR(rq))
	1760	+ return rq;
	1761	+
	1762	+ err = 0;
	1763	+ if (rq->engine->emit_init_breadcrumb) /* detect a hang */
	1764	+ err = rq->engine->emit_init_breadcrumb(rq);
	1765	+ if (err == 0)
	1766	+ err = rq->engine->emit_bb_start(rq, addr, 0, 0);
	1767	+
	1768	+ if (err == 0)
	1769	+ i915_request_get(rq);
	1770	+ i915_request_add(rq);
	1771	+
	1772	+ return err ? ERR_PTR(err) : rq;
	1773	+}
	1774	+
	1775	+static u32 spinner(u32 batch, int i)
	1776	+{
	1777	+ return batch + i * 64 / sizeof(*batch) + 4;
	1778	+}
	1779	+
	1780	+static void end_spin(u32 *batch, int i)
	1781	+{
	1782	+ *spinner(batch, i) = MI_BATCH_BUFFER_END;
	1783	+ wmb();
	1784	+}
	1785	+
	1786	+static int igt_cs_tlb(void *arg)
	1787	+{
	1788	+ const unsigned int count = PAGE_SIZE / 64;
	1789	+ const unsigned int chunk_size = count * PAGE_SIZE;
	1790	+ struct drm_i915_private *i915 = arg;
	1791	+ struct drm_i915_gem_object bbe, act, *out;
	1792	+ struct i915_gem_engines_iter it;
	1793	+ struct i915_address_space *vm;
	1794	+ struct i915_gem_context *ctx;
	1795	+ struct intel_context *ce;
	1796	+ struct i915_vma *vma;
	1797	+ I915_RND_STATE(prng);
	1798	+ struct file *file;
	1799	+ unsigned int i;
	1800	+ u32 *result;
	1801	+ u32 *batch;
	1802	+ int err = 0;
	1803	+
	1804	+ /*
	1805	+ * Our mission here is to fool the hardware to execute something
	1806	+ * from scratch as it has not seen the batch move (due to missing
	1807	+ * the TLB invalidate).
	1808	+ */
	1809	+
	1810	+ file = mock_file(i915);
	1811	+ if (IS_ERR(file))
	1812	+ return PTR_ERR(file);
	1813	+
	1814	+ ctx = live_context(i915, file);
	1815	+ if (IS_ERR(ctx)) {
	1816	+ err = PTR_ERR(ctx);
	1817	+ goto out_unlock;
	1818	+ }
	1819	+
	1820	+ vm = i915_gem_context_get_vm_rcu(ctx);
	1821	+ if (i915_is_ggtt(vm))
	1822	+ goto out_vm;
	1823	+
	1824	+ /* Create two pages; dummy we prefill the TLB, and intended */
	1825	+ bbe = i915_gem_object_create_internal(i915, PAGE_SIZE);
	1826	+ if (IS_ERR(bbe)) {
	1827	+ err = PTR_ERR(bbe);
	1828	+ goto out_vm;
	1829	+ }
	1830	+
	1831	+ batch = i915_gem_object_pin_map(bbe, I915_MAP_WC);
	1832	+ if (IS_ERR(batch)) {
	1833	+ err = PTR_ERR(batch);
	1834	+ goto out_put_bbe;
	1835	+ }
	1836	+ memset32(batch, MI_BATCH_BUFFER_END, PAGE_SIZE / sizeof(u32));
	1837	+ i915_gem_object_flush_map(bbe);
	1838	+ i915_gem_object_unpin_map(bbe);
	1839	+
	1840	+ act = i915_gem_object_create_internal(i915, PAGE_SIZE);
	1841	+ if (IS_ERR(act)) {
	1842	+ err = PTR_ERR(act);
	1843	+ goto out_put_bbe;
	1844	+ }
	1845	+
	1846	+ /* Track the execution of each request by writing into different slot */
	1847	+ batch = i915_gem_object_pin_map(act, I915_MAP_WC);
	1848	+ if (IS_ERR(batch)) {
	1849	+ err = PTR_ERR(batch);
	1850	+ goto out_put_act;
	1851	+ }
	1852	+ for (i = 0; i < count; i++) {
	1853	+ u32 cs = batch + i 64 / sizeof(*cs);
	1854	+ u64 addr = (vm->total - PAGE_SIZE) + i * sizeof(u32);
	1855	+
	1856	+ GEM_BUG_ON(INTEL_GEN(i915) < 6);
	1857	+ cs[0] = MI_STORE_DWORD_IMM_GEN4;
	1858	+ if (INTEL_GEN(i915) >= 8) {
	1859	+ cs[1] = lower_32_bits(addr);
	1860	+ cs[2] = upper_32_bits(addr);
	1861	+ cs[3] = i;
	1862	+ cs[4] = MI_NOOP;
	1863	+ cs[5] = MI_BATCH_BUFFER_START_GEN8;
	1864	+ } else {
	1865	+ cs[1] = 0;
	1866	+ cs[2] = lower_32_bits(addr);
	1867	+ cs[3] = i;
	1868	+ cs[4] = MI_NOOP;
	1869	+ cs[5] = MI_BATCH_BUFFER_START;
	1870	+ }
	1871	+ }
	1872	+
	1873	+ out = i915_gem_object_create_internal(i915, PAGE_SIZE);
	1874	+ if (IS_ERR(out)) {
	1875	+ err = PTR_ERR(out);
	1876	+ goto out_put_batch;
	1877	+ }
	1878	+ i915_gem_object_set_cache_coherency(out, I915_CACHING_CACHED);
	1879	+
	1880	+ vma = i915_vma_instance(out, vm, NULL);
	1881	+ if (IS_ERR(vma)) {
	1882	+ err = PTR_ERR(vma);
	1883	+ goto out_put_out;
	1884	+ }
	1885	+
	1886	+ err = i915_vma_pin(vma, 0, 0,
	1887	+ PIN_USER \|
	1888	+ PIN_OFFSET_FIXED \|
	1889	+ (vm->total - PAGE_SIZE));
	1890	+ if (err)
	1891	+ goto out_put_out;
	1892	+ GEM_BUG_ON(vma->node.start != vm->total - PAGE_SIZE);
	1893	+
	1894	+ result = i915_gem_object_pin_map(out, I915_MAP_WB);
	1895	+ if (IS_ERR(result)) {
	1896	+ err = PTR_ERR(result);
	1897	+ goto out_put_out;
	1898	+ }
	1899	+
	1900	+ for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) {
	1901	+ IGT_TIMEOUT(end_time);
	1902	+ unsigned long pass = 0;
	1903	+
	1904	+ if (!intel_engine_can_store_dword(ce->engine))
	1905	+ continue;
	1906	+
	1907	+ while (!__igt_timeout(end_time, NULL)) {
	1908	+ struct i915_vm_pt_stash stash = {};
	1909	+ struct i915_request *rq;
	1910	+ u64 offset;
	1911	+
	1912	+ offset = igt_random_offset(&prng,
	1913	+ 0, vm->total - PAGE_SIZE,
	1914	+ chunk_size, PAGE_SIZE);
	1915	+
	1916	+ memset32(result, STACK_MAGIC, PAGE_SIZE / sizeof(u32));
	1917	+
	1918	+ vma = i915_vma_instance(bbe, vm, NULL);
	1919	+ if (IS_ERR(vma)) {
	1920	+ err = PTR_ERR(vma);
	1921	+ goto end;
	1922	+ }
	1923	+
	1924	+ err = vma->ops->set_pages(vma);
	1925	+ if (err)
	1926	+ goto end;
	1927	+
	1928	+ err = i915_vm_alloc_pt_stash(vm, &stash, chunk_size);
	1929	+ if (err)
	1930	+ goto end;
	1931	+
	1932	+ err = i915_vm_pin_pt_stash(vm, &stash);
	1933	+ if (err) {
	1934	+ i915_vm_free_pt_stash(vm, &stash);
	1935	+ goto end;
	1936	+ }
	1937	+
	1938	+ vm->allocate_va_range(vm, &stash, offset, chunk_size);
	1939	+
	1940	+ i915_vm_free_pt_stash(vm, &stash);
	1941	+
	1942	+ /* Prime the TLB with the dummy pages */
	1943	+ for (i = 0; i < count; i++) {
	1944	+ vma->node.start = offset + i * PAGE_SIZE;
	1945	+ vm->insert_entries(vm, vma, I915_CACHE_NONE, 0);
	1946	+
	1947	+ rq = submit_batch(ce, vma->node.start);
	1948	+ if (IS_ERR(rq)) {
	1949	+ err = PTR_ERR(rq);
	1950	+ goto end;
	1951	+ }
	1952	+ i915_request_put(rq);
	1953	+ }
	1954	+
	1955	+ vma->ops->clear_pages(vma);
	1956	+
	1957	+ err = context_sync(ce);
	1958	+ if (err) {
	1959	+ pr_err("%s: dummy setup timed out\n",
	1960	+ ce->engine->name);
	1961	+ goto end;
	1962	+ }
	1963	+
	1964	+ vma = i915_vma_instance(act, vm, NULL);
	1965	+ if (IS_ERR(vma)) {
	1966	+ err = PTR_ERR(vma);
	1967	+ goto end;
	1968	+ }
	1969	+
	1970	+ err = vma->ops->set_pages(vma);
	1971	+ if (err)
	1972	+ goto end;
	1973	+
	1974	+ /* Replace the TLB with target batches */
	1975	+ for (i = 0; i < count; i++) {
	1976	+ struct i915_request *rq;
	1977	+ u32 cs = batch + i 64 / sizeof(*cs);
	1978	+ u64 addr;
	1979	+
	1980	+ vma->node.start = offset + i * PAGE_SIZE;
	1981	+ vm->insert_entries(vm, vma, I915_CACHE_NONE, 0);
	1982	+
	1983	+ addr = vma->node.start + i * 64;
	1984	+ cs[4] = MI_NOOP;
	1985	+ cs[6] = lower_32_bits(addr);
	1986	+ cs[7] = upper_32_bits(addr);
	1987	+ wmb();
	1988	+
	1989	+ rq = submit_batch(ce, addr);
	1990	+ if (IS_ERR(rq)) {
	1991	+ err = PTR_ERR(rq);
	1992	+ goto end;
	1993	+ }
	1994	+
	1995	+ /* Wait until the context chain has started */
	1996	+ if (i == 0) {
	1997	+ while (READ_ONCE(result[i]) &&
	1998	+ !i915_request_completed(rq))
	1999	+ cond_resched();
	2000	+ } else {
	2001	+ end_spin(batch, i - 1);
	2002	+ }
	2003	+
	2004	+ i915_request_put(rq);
	2005	+ }
	2006	+ end_spin(batch, count - 1);
	2007	+
	2008	+ vma->ops->clear_pages(vma);
	2009	+
	2010	+ err = context_sync(ce);
	2011	+ if (err) {
	2012	+ pr_err("%s: writes timed out\n",
	2013	+ ce->engine->name);
	2014	+ goto end;
	2015	+ }
	2016	+
	2017	+ for (i = 0; i < count; i++) {
	2018	+ if (result[i] != i) {
	2019	+ pr_err("%s: Write lost on pass %lu, at offset %llx, index %d, found %x, expected %x\n",
	2020	+ ce->engine->name, pass,
	2021	+ offset, i, result[i], i);
	2022	+ err = -EINVAL;
	2023	+ goto end;
	2024	+ }
	2025	+ }
	2026	+
	2027	+ vm->clear_range(vm, offset, chunk_size);
	2028	+ pass++;
	2029	+ }
	2030	+ }
	2031	+end:
	2032	+ if (igt_flush_test(i915))
	2033	+ err = -EIO;
	2034	+ i915_gem_context_unlock_engines(ctx);
	2035	+ i915_gem_object_unpin_map(out);
	2036	+out_put_out:
	2037	+ i915_gem_object_put(out);
	2038	+out_put_batch:
	2039	+ i915_gem_object_unpin_map(act);
	2040	+out_put_act:
	2041	+ i915_gem_object_put(act);
	2042	+out_put_bbe:
	2043	+ i915_gem_object_put(bbe);
	2044	+out_vm:
	2045	+ i915_vm_put(vm);
	2046	+out_unlock:
	2047	+ fput(file);
1678	2048	return err;
1679	2049	}
1680	2050
..	..	@@ -1695,6 +2065,7 @@
1695	2065	SUBTEST(igt_ggtt_pot),
1696	2066	SUBTEST(igt_ggtt_fill),
1697	2067	SUBTEST(igt_ggtt_page),
	2068	+ SUBTEST(igt_cs_tlb),
1698	2069	};
1699	2070
1700	2071	GEM_BUG_ON(offset_in_page(i915->ggtt.vm.total));