~hc/RK356X_SDK_RELEASE.git

..	..	@@ -22,6 +22,7 @@
22	22
23	23	#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
24	24
	25	+#include <linux/dma-mapping.h>
25	26	#include <linux/module.h>
26	27	#include <linux/kernel.h>
27	28	#include <linux/init.h>
..	..	@@ -34,9 +35,7 @@
34	35	#include <linux/slab.h>
35	36	#include <linux/highmem.h>
36	37	#include <linux/refcount.h>
37		-#ifdef CONFIG_XEN_GRANT_DMA_ALLOC
38		-#include <linux/of_device.h>
39		-#endif
	38	+#include <linux/workqueue.h>
40	39
41	40	#include <xen/xen.h>
42	41	#include <xen/grant_table.h>
..	..	@@ -57,26 +56,24 @@
57	56	"Gerd Hoffmann <kraxel@redhat.com>");
58	57	MODULE_DESCRIPTION("User-space granted page access driver");
59	58
60		-static int limit = 1024*1024;
61		-module_param(limit, int, 0644);
62		-MODULE_PARM_DESC(limit, "Maximum number of grants that may be mapped by "
63		- "the gntdev device");
	59	+static unsigned int limit = 64*1024;
	60	+module_param(limit, uint, 0644);
	61	+MODULE_PARM_DESC(limit,
	62	+ "Maximum number of grants that may be mapped by one mapping request");
64	63
65		-static atomic_t pages_mapped = ATOMIC_INIT(0);
66		-
	64	+/* True in PV mode, false otherwise */
67	65	static int use_ptemod;
68		-#define populate_freeable_maps use_ptemod
69	66
70		-static int unmap_grant_pages(struct gntdev_grant_map *map,
71		- int offset, int pages);
	67	+static void unmap_grant_pages(struct gntdev_grant_map *map,
	68	+ int offset, int pages);
72	69
73	70	static struct miscdevice gntdev_miscdev;
74	71
75	72	/* ------------------------------------------------------------------ */
76	73
77		-bool gntdev_account_mapped_pages(int count)
	74	+bool gntdev_test_page_count(unsigned int count)
78	75	{
79		- return atomic_add_return(count, &pages_mapped) > limit;
	76	+ return !count \|\| count > limit;
80	77	}
81	78
82	79	static void gntdev_print_maps(struct gntdev_priv *priv,
..	..	@@ -117,14 +114,15 @@
117	114	gnttab_free_pages(map->count, map->pages);
118	115
119	116	#ifdef CONFIG_XEN_GRANT_DMA_ALLOC
120		- kfree(map->frames);
	117	+ kvfree(map->frames);
121	118	#endif
122		- kfree(map->pages);
123		- kfree(map->grants);
124		- kfree(map->map_ops);
125		- kfree(map->unmap_ops);
126		- kfree(map->kmap_ops);
127		- kfree(map->kunmap_ops);
	119	+ kvfree(map->pages);
	120	+ kvfree(map->grants);
	121	+ kvfree(map->map_ops);
	122	+ kvfree(map->unmap_ops);
	123	+ kvfree(map->kmap_ops);
	124	+ kvfree(map->kunmap_ops);
	125	+ kvfree(map->being_removed);
128	126	kfree(map);
129	127	}
130	128
..	..	@@ -138,18 +136,22 @@
138	136	if (NULL == add)
139	137	return NULL;
140	138
141		- add->grants = kcalloc(count, sizeof(add->grants[0]), GFP_KERNEL);
142		- add->map_ops = kcalloc(count, sizeof(add->map_ops[0]), GFP_KERNEL);
143		- add->unmap_ops = kcalloc(count, sizeof(add->unmap_ops[0]), GFP_KERNEL);
144		- add->kmap_ops = kcalloc(count, sizeof(add->kmap_ops[0]), GFP_KERNEL);
145		- add->kunmap_ops = kcalloc(count, sizeof(add->kunmap_ops[0]), GFP_KERNEL);
146		- add->pages = kcalloc(count, sizeof(add->pages[0]), GFP_KERNEL);
	139	+ add->grants = kvcalloc(count, sizeof(add->grants[0]), GFP_KERNEL);
	140	+ add->map_ops = kvcalloc(count, sizeof(add->map_ops[0]), GFP_KERNEL);
	141	+ add->unmap_ops = kvcalloc(count, sizeof(add->unmap_ops[0]), GFP_KERNEL);
	142	+ add->kmap_ops = kvcalloc(count, sizeof(add->kmap_ops[0]), GFP_KERNEL);
	143	+ add->kunmap_ops = kvcalloc(count,
	144	+ sizeof(add->kunmap_ops[0]), GFP_KERNEL);
	145	+ add->pages = kvcalloc(count, sizeof(add->pages[0]), GFP_KERNEL);
	146	+ add->being_removed =
	147	+ kvcalloc(count, sizeof(add->being_removed[0]), GFP_KERNEL);
147	148	if (NULL == add->grants \|\|
148	149	NULL == add->map_ops \|\|
149	150	NULL == add->unmap_ops \|\|
150	151	NULL == add->kmap_ops \|\|
151	152	NULL == add->kunmap_ops \|\|
152		- NULL == add->pages)
	153	+ NULL == add->pages \|\|
	154	+ NULL == add->being_removed)
153	155	goto err;
154	156
155	157	#ifdef CONFIG_XEN_GRANT_DMA_ALLOC
..	..	@@ -162,8 +164,8 @@
162	164	if (dma_flags & (GNTDEV_DMA_FLAG_WC \| GNTDEV_DMA_FLAG_COHERENT)) {
163	165	struct gnttab_dma_alloc_args args;
164	166
165		- add->frames = kcalloc(count, sizeof(add->frames[0]),
166		- GFP_KERNEL);
	167	+ add->frames = kvcalloc(count, sizeof(add->frames[0]),
	168	+ GFP_KERNEL);
167	169	if (!add->frames)
168	170	goto err;
169	171
..	..	@@ -244,44 +246,58 @@
244	246	if (!refcount_dec_and_test(&map->users))
245	247	return;
246	248
247		- atomic_sub(map->count, &pages_mapped);
	249	+ if (map->pages && !use_ptemod) {
	250	+ /*
	251	+ * Increment the reference count. This ensures that the
	252	+ * subsequent call to unmap_grant_pages() will not wind up
	253	+ * re-entering itself. It can wind up calling
	254	+ * gntdev_put_map() recursively, but such calls will be with a
	255	+ * reference count greater than 1, so they will return before
	256	+ * this code is reached. The recursion depth is thus limited to
	257	+ * 1. Do NOT use refcount_inc() here, as it will detect that
	258	+ * the reference count is zero and WARN().
	259	+ */
	260	+ refcount_set(&map->users, 1);
	261	+
	262	+ /*
	263	+ * Unmap the grants. This may or may not be asynchronous, so it
	264	+ * is possible that the reference count is 1 on return, but it
	265	+ * could also be greater than 1.
	266	+ */
	267	+ unmap_grant_pages(map, 0, map->count);
	268	+
	269	+ /* Check if the memory now needs to be freed */
	270	+ if (!refcount_dec_and_test(&map->users))
	271	+ return;
	272	+
	273	+ /*
	274	+ * All pages have been returned to the hypervisor, so free the
	275	+ * map.
	276	+ */
	277	+ }
	278	+
	279	+ if (use_ptemod && map->notifier_init)
	280	+ mmu_interval_notifier_remove(&map->notifier);
248	281
249	282	if (map->notify.flags & UNMAP_NOTIFY_SEND_EVENT) {
250	283	notify_remote_via_evtchn(map->notify.event);
251	284	evtchn_put(map->notify.event);
252	285	}
253		-
254		- if (populate_freeable_maps && priv) {
255		- mutex_lock(&priv->lock);
256		- list_del(&map->next);
257		- mutex_unlock(&priv->lock);
258		- }
259		-
260		- if (map->pages && !use_ptemod)
261		- unmap_grant_pages(map, 0, map->count);
262	286	gntdev_free_map(map);
263	287	}
264	288
265	289	/* ------------------------------------------------------------------ */
266	290
267		-static int find_grant_ptes(pte_t *pte, pgtable_t token,
268		- unsigned long addr, void *data)
	291	+static int find_grant_ptes(pte_t pte, unsigned long addr, void data)
269	292	{
270	293	struct gntdev_grant_map *map = data;
271		- unsigned int pgnr = (addr - map->vma->vm_start) >> PAGE_SHIFT;
272		- int flags = map->flags \| GNTMAP_application_map \| GNTMAP_contains_pte;
	294	+ unsigned int pgnr = (addr - map->pages_vm_start) >> PAGE_SHIFT;
	295	+ int flags = map->flags \| GNTMAP_application_map \| GNTMAP_contains_pte \|
	296	+ (1 << _GNTMAP_guest_avail0);
273	297	u64 pte_maddr;
274	298
275	299	BUG_ON(pgnr >= map->count);
276	300	pte_maddr = arbitrary_virt_to_machine(pte).maddr;
277		-
278		- /*
279		- * Set the PTE as special to force get_user_pages_fast() fall
280		- * back to the slow path. If this is not supported as part of
281		- * the grant map, it will be done afterwards.
282		- */
283		- if (xen_feature(XENFEAT_gnttab_map_avail_bits))
284		- flags \|= (1 << _GNTMAP_guest_avail0);
285	301
286	302	gnttab_set_map_op(&map->map_ops[pgnr], pte_maddr, flags,
287	303	map->grants[pgnr].ref,
..	..	@@ -291,17 +307,9 @@
291	307	return 0;
292	308	}
293	309
294		-#ifdef CONFIG_X86
295		-static int set_grant_ptes_as_special(pte_t *pte, pgtable_t token,
296		- unsigned long addr, void *data)
297		-{
298		- set_pte_at(current->mm, addr, pte, pte_mkspecial(*pte));
299		- return 0;
300		-}
301		-#endif
302		-
303	310	int gntdev_map_grant_pages(struct gntdev_grant_map *map)
304	311	{
	312	+ size_t alloced = 0;
305	313	int i, err = 0;
306	314
307	315	if (!use_ptemod) {
..	..	@@ -350,87 +358,130 @@
350	358	map->pages, map->count);
351	359
352	360	for (i = 0; i < map->count; i++) {
353		- if (map->map_ops[i].status == GNTST_okay)
	361	+ if (map->map_ops[i].status == GNTST_okay) {
354	362	map->unmap_ops[i].handle = map->map_ops[i].handle;
355		- else if (!err)
	363	+ alloced++;
	364	+ } else if (!err)
356	365	err = -EINVAL;
357	366
358	367	if (map->flags & GNTMAP_device_map)
359	368	map->unmap_ops[i].dev_bus_addr = map->map_ops[i].dev_bus_addr;
360	369
361	370	if (use_ptemod) {
362		- if (map->kmap_ops[i].status == GNTST_okay)
	371	+ if (map->kmap_ops[i].status == GNTST_okay) {
	372	+ alloced++;
363	373	map->kunmap_ops[i].handle = map->kmap_ops[i].handle;
364		- else if (!err)
	374	+ } else if (!err)
365	375	err = -EINVAL;
366	376	}
367	377	}
	378	+ atomic_add(alloced, &map->live_grants);
368	379	return err;
369	380	}
370	381
371		-static int __unmap_grant_pages(struct gntdev_grant_map *map, int offset,
	382	+static void __unmap_grant_pages_done(int result,
	383	+ struct gntab_unmap_queue_data *data)
	384	+{
	385	+ unsigned int i;
	386	+ struct gntdev_grant_map *map = data->data;
	387	+ unsigned int offset = data->unmap_ops - map->unmap_ops;
	388	+ int successful_unmaps = 0;
	389	+ int live_grants;
	390	+
	391	+ for (i = 0; i < data->count; i++) {
	392	+ if (map->unmap_ops[offset + i].status == GNTST_okay &&
	393	+ map->unmap_ops[offset + i].handle != -1)
	394	+ successful_unmaps++;
	395	+
	396	+ WARN_ON(map->unmap_ops[offset+i].status &&
	397	+ map->unmap_ops[offset+i].handle != -1);
	398	+ pr_debug("unmap handle=%d st=%d\n",
	399	+ map->unmap_ops[offset+i].handle,
	400	+ map->unmap_ops[offset+i].status);
	401	+ map->unmap_ops[offset+i].handle = -1;
	402	+ if (use_ptemod) {
	403	+ if (map->kunmap_ops[offset + i].status == GNTST_okay &&
	404	+ map->kunmap_ops[offset + i].handle != -1)
	405	+ successful_unmaps++;
	406	+
	407	+ WARN_ON(map->kunmap_ops[offset+i].status &&
	408	+ map->kunmap_ops[offset+i].handle != -1);
	409	+ pr_debug("kunmap handle=%u st=%d\n",
	410	+ map->kunmap_ops[offset+i].handle,
	411	+ map->kunmap_ops[offset+i].status);
	412	+ map->kunmap_ops[offset+i].handle = -1;
	413	+ }
	414	+ }
	415	+
	416	+ /*
	417	+ * Decrease the live-grant counter. This must happen after the loop to
	418	+ * prevent premature reuse of the grants by gnttab_mmap().
	419	+ */
	420	+ live_grants = atomic_sub_return(successful_unmaps, &map->live_grants);
	421	+ if (WARN_ON(live_grants < 0))
	422	+ pr_err("%s: live_grants became negative (%d) after unmapping %d pages!\n",
	423	+ __func__, live_grants, successful_unmaps);
	424	+
	425	+ /* Release reference taken by __unmap_grant_pages */
	426	+ gntdev_put_map(NULL, map);
	427	+}
	428	+
	429	+static void __unmap_grant_pages(struct gntdev_grant_map *map, int offset,
372	430	int pages)
373	431	{
374		- int i, err = 0;
375		- struct gntab_unmap_queue_data unmap_data;
376		-
377	432	if (map->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) {
378	433	int pgno = (map->notify.addr >> PAGE_SHIFT);
	434	+
379	435	if (pgno >= offset && pgno < offset + pages) {
380	436	/* No need for kmap, pages are in lowmem */
381	437	uint8_t *tmp = pfn_to_kaddr(page_to_pfn(map->pages[pgno]));
	438	+
382	439	tmp[map->notify.addr & (PAGE_SIZE-1)] = 0;
383	440	map->notify.flags &= ~UNMAP_NOTIFY_CLEAR_BYTE;
384	441	}
385	442	}
386	443
387		- unmap_data.unmap_ops = map->unmap_ops + offset;
388		- unmap_data.kunmap_ops = use_ptemod ? map->kunmap_ops + offset : NULL;
389		- unmap_data.pages = map->pages + offset;
390		- unmap_data.count = pages;
	444	+ map->unmap_data.unmap_ops = map->unmap_ops + offset;
	445	+ map->unmap_data.kunmap_ops = use_ptemod ? map->kunmap_ops + offset : NULL;
	446	+ map->unmap_data.pages = map->pages + offset;
	447	+ map->unmap_data.count = pages;
	448	+ map->unmap_data.done = __unmap_grant_pages_done;
	449	+ map->unmap_data.data = map;
	450	+ refcount_inc(&map->users); /* to keep map alive during async call below */
391	451
392		- err = gnttab_unmap_refs_sync(&unmap_data);
393		- if (err)
394		- return err;
395		-
396		- for (i = 0; i < pages; i++) {
397		- if (map->unmap_ops[offset+i].status)
398		- err = -EINVAL;
399		- pr_debug("unmap handle=%d st=%d\n",
400		- map->unmap_ops[offset+i].handle,
401		- map->unmap_ops[offset+i].status);
402		- map->unmap_ops[offset+i].handle = -1;
403		- }
404		- return err;
	452	+ gnttab_unmap_refs_async(&map->unmap_data);
405	453	}
406	454
407		-static int unmap_grant_pages(struct gntdev_grant_map *map, int offset,
408		- int pages)
	455	+static void unmap_grant_pages(struct gntdev_grant_map *map, int offset,
	456	+ int pages)
409	457	{
410		- int range, err = 0;
	458	+ int range;
	459	+
	460	+ if (atomic_read(&map->live_grants) == 0)
	461	+ return; /* Nothing to do */
411	462
412	463	pr_debug("unmap %d+%d [%d+%d]\n", map->index, map->count, offset, pages);
413	464
414	465	/* It is possible the requested range will have a "hole" where we
415	466	* already unmapped some of the grants. Only unmap valid ranges.
416	467	*/
417		- while (pages && !err) {
418		- while (pages && map->unmap_ops[offset].handle == -1) {
	468	+ while (pages) {
	469	+ while (pages && map->being_removed[offset]) {
419	470	offset++;
420	471	pages--;
421	472	}
422	473	range = 0;
423	474	while (range < pages) {
424		- if (map->unmap_ops[offset+range].handle == -1)
	475	+ if (map->being_removed[offset + range])
425	476	break;
	477	+ map->being_removed[offset + range] = true;
426	478	range++;
427	479	}
428		- err = __unmap_grant_pages(map, offset, range);
	480	+ if (range)
	481	+ __unmap_grant_pages(map, offset, range);
429	482	offset += range;
430	483	pages -= range;
431	484	}
432		-
433		- return err;
434	485	}
435	486
436	487	/* ------------------------------------------------------------------ */
..	..	@@ -450,18 +501,7 @@
450	501	struct gntdev_priv *priv = file->private_data;
451	502
452	503	pr_debug("gntdev_vma_close %p\n", vma);
453		- if (use_ptemod) {
454		- /* It is possible that an mmu notifier could be running
455		- * concurrently, so take priv->lock to ensure that the vma won't
456		- * vanishing during the unmap_grant_pages call, since we will
457		- * spin here until that completes. Such a concurrent call will
458		- * not do any unmapping, since that has been done prior to
459		- * closing the vma, but it may still iterate the unmap_ops list.
460		- */
461		- mutex_lock(&priv->lock);
462		- map->vma = NULL;
463		- mutex_unlock(&priv->lock);
464		- }
	504	+
465	505	vma->vm_private_data = NULL;
466	506	gntdev_put_map(priv, map);
467	507	}
..	..	@@ -482,109 +522,43 @@
482	522
483	523	/* ------------------------------------------------------------------ */
484	524
485		-static bool in_range(struct gntdev_grant_map *map,
486		- unsigned long start, unsigned long end)
	525	+static bool gntdev_invalidate(struct mmu_interval_notifier *mn,
	526	+ const struct mmu_notifier_range *range,
	527	+ unsigned long cur_seq)
487	528	{
488		- if (!map->vma)
	529	+ struct gntdev_grant_map *map =
	530	+ container_of(mn, struct gntdev_grant_map, notifier);
	531	+ unsigned long mstart, mend;
	532	+ unsigned long map_start, map_end;
	533	+
	534	+ if (!mmu_notifier_range_blockable(range))
489	535	return false;
490		- if (map->vma->vm_start >= end)
491		- return false;
492		- if (map->vma->vm_end <= start)
493		- return false;
	536	+
	537	+ map_start = map->pages_vm_start;
	538	+ map_end = map->pages_vm_start + (map->count << PAGE_SHIFT);
	539	+
	540	+ /*
	541	+ * If the VMA is split or otherwise changed the notifier is not
	542	+ * updated, but we don't want to process VA's outside the modified
	543	+ * VMA. FIXME: It would be much more understandable to just prevent
	544	+ * modifying the VMA in the first place.
	545	+ */
	546	+ if (map_start >= range->end \|\| map_end <= range->start)
	547	+ return true;
	548	+
	549	+ mstart = max(range->start, map_start);
	550	+ mend = min(range->end, map_end);
	551	+ pr_debug("map %d+%d (%lx %lx), range %lx %lx, mrange %lx %lx\n",
	552	+ map->index, map->count, map_start, map_end,
	553	+ range->start, range->end, mstart, mend);
	554	+ unmap_grant_pages(map, (mstart - map_start) >> PAGE_SHIFT,
	555	+ (mend - mstart) >> PAGE_SHIFT);
494	556
495	557	return true;
496	558	}
497	559
498		-static int unmap_if_in_range(struct gntdev_grant_map *map,
499		- unsigned long start, unsigned long end,
500		- bool blockable)
501		-{
502		- unsigned long mstart, mend;
503		- int err;
504		-
505		- if (!in_range(map, start, end))
506		- return 0;
507		-
508		- if (!blockable)
509		- return -EAGAIN;
510		-
511		- mstart = max(start, map->vma->vm_start);
512		- mend = min(end, map->vma->vm_end);
513		- pr_debug("map %d+%d (%lx %lx), range %lx %lx, mrange %lx %lx\n",
514		- map->index, map->count,
515		- map->vma->vm_start, map->vma->vm_end,
516		- start, end, mstart, mend);
517		- err = unmap_grant_pages(map,
518		- (mstart - map->vma->vm_start) >> PAGE_SHIFT,
519		- (mend - mstart) >> PAGE_SHIFT);
520		- WARN_ON(err);
521		-
522		- return 0;
523		-}
524		-
525		-static int mn_invl_range_start(struct mmu_notifier *mn,
526		- struct mm_struct *mm,
527		- unsigned long start, unsigned long end,
528		- bool blockable)
529		-{
530		- struct gntdev_priv *priv = container_of(mn, struct gntdev_priv, mn);
531		- struct gntdev_grant_map *map;
532		- int ret = 0;
533		-
534		- if (blockable)
535		- mutex_lock(&priv->lock);
536		- else if (!mutex_trylock(&priv->lock))
537		- return -EAGAIN;
538		-
539		- list_for_each_entry(map, &priv->maps, next) {
540		- ret = unmap_if_in_range(map, start, end, blockable);
541		- if (ret)
542		- goto out_unlock;
543		- }
544		- list_for_each_entry(map, &priv->freeable_maps, next) {
545		- ret = unmap_if_in_range(map, start, end, blockable);
546		- if (ret)
547		- goto out_unlock;
548		- }
549		-
550		-out_unlock:
551		- mutex_unlock(&priv->lock);
552		-
553		- return ret;
554		-}
555		-
556		-static void mn_release(struct mmu_notifier *mn,
557		- struct mm_struct *mm)
558		-{
559		- struct gntdev_priv *priv = container_of(mn, struct gntdev_priv, mn);
560		- struct gntdev_grant_map *map;
561		- int err;
562		-
563		- mutex_lock(&priv->lock);
564		- list_for_each_entry(map, &priv->maps, next) {
565		- if (!map->vma)
566		- continue;
567		- pr_debug("map %d+%d (%lx %lx)\n",
568		- map->index, map->count,
569		- map->vma->vm_start, map->vma->vm_end);
570		- err = unmap_grant_pages(map, /* offset */ 0, map->count);
571		- WARN_ON(err);
572		- }
573		- list_for_each_entry(map, &priv->freeable_maps, next) {
574		- if (!map->vma)
575		- continue;
576		- pr_debug("map %d+%d (%lx %lx)\n",
577		- map->index, map->count,
578		- map->vma->vm_start, map->vma->vm_end);
579		- err = unmap_grant_pages(map, /* offset */ 0, map->count);
580		- WARN_ON(err);
581		- }
582		- mutex_unlock(&priv->lock);
583		-}
584		-
585		-static const struct mmu_notifier_ops gntdev_mmu_ops = {
586		- .release = mn_release,
587		- .invalidate_range_start = mn_invl_range_start,
	560	+static const struct mmu_interval_notifier_ops gntdev_mmu_ops = {
	561	+ .invalidate = gntdev_invalidate,
588	562	};
589	563
590	564	/* ------------------------------------------------------------------ */
..	..	@@ -592,52 +566,28 @@
592	566	static int gntdev_open(struct inode inode, struct file flip)
593	567	{
594	568	struct gntdev_priv *priv;
595		- int ret = 0;
596	569
597	570	priv = kzalloc(sizeof(*priv), GFP_KERNEL);
598	571	if (!priv)
599	572	return -ENOMEM;
600	573
601	574	INIT_LIST_HEAD(&priv->maps);
602		- INIT_LIST_HEAD(&priv->freeable_maps);
603	575	mutex_init(&priv->lock);
604	576
605	577	#ifdef CONFIG_XEN_GNTDEV_DMABUF
606	578	priv->dmabuf_priv = gntdev_dmabuf_init(flip);
607	579	if (IS_ERR(priv->dmabuf_priv)) {
608		- ret = PTR_ERR(priv->dmabuf_priv);
	580	+ int ret = PTR_ERR(priv->dmabuf_priv);
	581	+
609	582	kfree(priv);
610	583	return ret;
611	584	}
612	585	#endif
613	586
614		- if (use_ptemod) {
615		- priv->mm = get_task_mm(current);
616		- if (!priv->mm) {
617		- kfree(priv);
618		- return -ENOMEM;
619		- }
620		- priv->mn.ops = &gntdev_mmu_ops;
621		- ret = mmu_notifier_register(&priv->mn, priv->mm);
622		- mmput(priv->mm);
623		- }
624		-
625		- if (ret) {
626		- kfree(priv);
627		- return ret;
628		- }
629		-
630	587	flip->private_data = priv;
631	588	#ifdef CONFIG_XEN_GRANT_DMA_ALLOC
632	589	priv->dma_dev = gntdev_miscdev.this_device;
633		-
634		- /*
635		- * The device is not spawn from a device tree, so arch_setup_dma_ops
636		- * is not called, thus leaving the device with dummy DMA ops.
637		- * Fix this by calling of_dma_configure() with a NULL node to set
638		- * default DMA ops.
639		- */
640		- of_dma_configure(priv->dma_dev, NULL, true);
	590	+ dma_coerce_mask_and_coherent(priv->dma_dev, DMA_BIT_MASK(64));
641	591	#endif
642	592	pr_debug("priv %p\n", priv);
643	593
..	..	@@ -658,15 +608,11 @@
658	608	list_del(&map->next);
659	609	gntdev_put_map(NULL /* already removed */, map);
660	610	}
661		- WARN_ON(!list_empty(&priv->freeable_maps));
662	611	mutex_unlock(&priv->lock);
663	612
664	613	#ifdef CONFIG_XEN_GNTDEV_DMABUF
665	614	gntdev_dmabuf_fini(priv->dmabuf_priv);
666	615	#endif
667		-
668		- if (use_ptemod)
669		- mmu_notifier_unregister(&priv->mn, priv->mm);
670	616
671	617	kfree(priv);
672	618	return 0;
..	..	@@ -682,19 +628,13 @@
682	628	if (copy_from_user(&op, u, sizeof(op)) != 0)
683	629	return -EFAULT;
684	630	pr_debug("priv %p, add %d\n", priv, op.count);
685		- if (unlikely(op.count <= 0))
	631	+ if (unlikely(gntdev_test_page_count(op.count)))
686	632	return -EINVAL;
687	633
688	634	err = -ENOMEM;
689	635	map = gntdev_alloc_map(priv, op.count, 0 /* This is not a dma-buf. */);
690	636	if (!map)
691	637	return err;
692		-
693		- if (unlikely(gntdev_account_mapped_pages(op.count))) {
694		- pr_debug("can't map: over limit\n");
695		- gntdev_put_map(NULL, map);
696		- return err;
697		- }
698	638
699	639	if (copy_from_user(map->grants, &u->refs,
700	640	sizeof(map->grants[0]) * op.count) != 0) {
..	..	@@ -728,8 +668,6 @@
728	668	map = gntdev_find_map_index(priv, op.index >> PAGE_SHIFT, op.count);
729	669	if (map) {
730	670	list_del(&map->next);
731		- if (populate_freeable_maps)
732		- list_add_tail(&map->next, &priv->freeable_maps);
733	671	err = 0;
734	672	}
735	673	mutex_unlock(&priv->lock);
..	..	@@ -750,7 +688,7 @@
750	688	return -EFAULT;
751	689	pr_debug("priv %p, offset for vaddr %lx\n", priv, (unsigned long)op.vaddr);
752	690
753		- down_read(&current->mm->mmap_sem);
	691	+ mmap_read_lock(current->mm);
754	692	vma = find_vma(current->mm, op.vaddr);
755	693	if (!vma \|\| vma->vm_ops != &gntdev_vmops)
756	694	goto out_unlock;
..	..	@@ -764,7 +702,7 @@
764	702	rv = 0;
765	703
766	704	out_unlock:
767		- up_read(&current->mm->mmap_sem);
	705	+ mmap_read_unlock(current->mm);
768	706
769	707	if (rv == 0 && copy_to_user(u, &op, sizeof(op)) != 0)
770	708	return -EFAULT;
..	..	@@ -777,7 +715,7 @@
777	715	struct gntdev_grant_map *map;
778	716	int rc;
779	717	int out_flags;
780		- unsigned int out_event;
	718	+ evtchn_port_t out_event;
781	719
782	720	if (copy_from_user(&op, u, sizeof(op)))
783	721	return -EFAULT;
..	..	@@ -856,7 +794,7 @@
856	794	unsigned long xen_pfn;
857	795	int ret;
858	796
859		- ret = get_user_pages_fast(addr, 1, batch->writeable, &page);
	797	+ ret = pin_user_pages_fast(addr, 1, batch->writeable ? FOLL_WRITE : 0, &page);
860	798	if (ret < 0)
861	799	return ret;
862	800
..	..	@@ -870,13 +808,7 @@
870	808
871	809	static void gntdev_put_pages(struct gntdev_copy_batch *batch)
872	810	{
873		- unsigned int i;
874		-
875		- for (i = 0; i < batch->nr_pages; i++) {
876		- if (batch->writeable && !PageDirty(batch->pages[i]))
877		- set_page_dirty_lock(batch->pages[i]);
878		- put_page(batch->pages[i]);
879		- }
	811	+ unpin_user_pages_dirty_lock(batch->pages, batch->nr_pages, batch->writeable);
880	812	batch->nr_pages = 0;
881	813	batch->writeable = false;
882	814	}
..	..	@@ -1094,24 +1026,20 @@
1094	1026	int index = vma->vm_pgoff;
1095	1027	int count = vma_pages(vma);
1096	1028	struct gntdev_grant_map *map;
1097		- int i, err = -EINVAL;
	1029	+ int err = -EINVAL;
1098	1030
1099	1031	if ((vma->vm_flags & VM_WRITE) && !(vma->vm_flags & VM_SHARED))
1100	1032	return -EINVAL;
1101	1033
1102	1034	pr_debug("map %d+%d at %lx (pgoff %lx)\n",
1103		- index, count, vma->vm_start, vma->vm_pgoff);
	1035	+ index, count, vma->vm_start, vma->vm_pgoff);
1104	1036
1105	1037	mutex_lock(&priv->lock);
1106	1038	map = gntdev_find_map_index(priv, index, count);
1107	1039	if (!map)
1108	1040	goto unlock_out;
1109		- if (use_ptemod && map->vma)
	1041	+ if (!atomic_add_unless(&map->in_use, 1, 1))
1110	1042	goto unlock_out;
1111		- if (use_ptemod && priv->mm != vma->vm_mm) {
1112		- pr_warn("Huh? Other mm?\n");
1113		- goto unlock_out;
1114		- }
1115	1043
1116	1044	refcount_inc(&map->users);
1117	1045
..	..	@@ -1123,10 +1051,6 @@
1123	1051	vma->vm_flags \|= VM_DONTCOPY;
1124	1052
1125	1053	vma->vm_private_data = map;
1126		-
1127		- if (use_ptemod)
1128		- map->vma = vma;
1129		-
1130	1054	if (map->flags) {
1131	1055	if ((vma->vm_flags & VM_WRITE) &&
1132	1056	(map->flags & GNTMAP_readonly))
..	..	@@ -1137,10 +1061,32 @@
1137	1061	map->flags \|= GNTMAP_readonly;
1138	1062	}
1139	1063
	1064	+ map->pages_vm_start = vma->vm_start;
	1065	+
	1066	+ if (use_ptemod) {
	1067	+ err = mmu_interval_notifier_insert_locked(
	1068	+ &map->notifier, vma->vm_mm, vma->vm_start,
	1069	+ vma->vm_end - vma->vm_start, &gntdev_mmu_ops);
	1070	+ if (err)
	1071	+ goto out_unlock_put;
	1072	+
	1073	+ map->notifier_init = true;
	1074	+ }
1140	1075	mutex_unlock(&priv->lock);
1141	1076
1142	1077	if (use_ptemod) {
1143		- map->pages_vm_start = vma->vm_start;
	1078	+ /*
	1079	+ * gntdev takes the address of the PTE in find_grant_ptes() and
	1080	+ * passes it to the hypervisor in gntdev_map_grant_pages(). The
	1081	+ * purpose of the notifier is to prevent the hypervisor pointer
	1082	+ * to the PTE from going stale.
	1083	+ *
	1084	+ * Since this vma's mappings can't be touched without the
	1085	+ * mmap_lock, and we are holding it now, there is no need for
	1086	+ * the notifier_range locking pattern.
	1087	+ */
	1088	+ mmu_interval_read_begin(&map->notifier);
	1089	+
1144	1090	err = apply_to_page_range(vma->vm_mm, vma->vm_start,
1145	1091	vma->vm_end - vma->vm_start,
1146	1092	find_grant_ptes, map);
..	..	@@ -1155,29 +1101,9 @@
1155	1101	goto out_put_map;
1156	1102
1157	1103	if (!use_ptemod) {
1158		- for (i = 0; i < count; i++) {
1159		- err = vm_insert_page(vma, vma->vm_start + i*PAGE_SIZE,
1160		- map->pages[i]);
1161		- if (err)
1162		- goto out_put_map;
1163		- }
1164		- } else {
1165		-#ifdef CONFIG_X86
1166		- /*
1167		- * If the PTEs were not made special by the grant map
1168		- * hypercall, do so here.
1169		- *
1170		- * This is racy since the mapping is already visible
1171		- * to userspace but userspace should be well-behaved
1172		- * enough to not touch it until the mmap() call
1173		- * returns.
1174		- */
1175		- if (!xen_feature(XENFEAT_gnttab_map_avail_bits)) {
1176		- apply_to_page_range(vma->vm_mm, vma->vm_start,
1177		- vma->vm_end - vma->vm_start,
1178		- set_grant_ptes_as_special, NULL);
1179		- }
1180		-#endif
	1104	+ err = vm_map_pages_zero(vma, map->pages, map->count);
	1105	+ if (err)
	1106	+ goto out_put_map;
1181	1107	}
1182	1108
1183	1109	return 0;
..	..	@@ -1189,10 +1115,8 @@
1189	1115	out_unlock_put:
1190	1116	mutex_unlock(&priv->lock);
1191	1117	out_put_map:
1192		- if (use_ptemod) {
1193		- map->vma = NULL;
	1118	+ if (use_ptemod)
1194	1119	unmap_grant_pages(map, 0, map->count);
1195		- }
1196	1120	gntdev_put_map(priv, map);
1197	1121	return err;
1198	1122	}