write in 30M

hc

2024-02-20 ea08eeccae9297f7aabd2ef7f0c2517ac4549acc

 kernel/drivers/xen/grant-table.c
..
..
@@ -33,7 +33,7 @@
33
33
 
34
34
 #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
35
35
 
36
-#include <linux/bootmem.h>
36
+#include <linux/memblock.h>
37
37
 #include <linux/sched.h>
38
38
 #include <linux/mm.h>
39
39
 #include <linux/slab.h>
..
..
@@ -64,7 +64,6 @@
64
64
 #include <asm/xen/hypercall.h>
65
65
 #include <asm/xen/interface.h>
66
66
 
67
-#include <asm/pgtable.h>
68
67
 #include <asm/sync_bitops.h>
69
68
 
70
69
 /* External tools reserve first few grant table entries. */
..
..
@@ -135,12 +134,9 @@
135
134
 	 */
136
135
 	unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref);
137
136
 	/*
138
-	 * Query the status of a grant entry. Ref parameter is reference of
139
-	 * queried grant entry, return value is the status of queried entry.
140
-	 * Detailed status(writing/reading) can be gotten from the return value
141
-	 * by bit operations.
137
+	 * Read the frame number related to a given grant reference.
142
138
 	 */
143
-	int (*query_foreign_access)(grant_ref_t ref);
139
+	unsigned long (*read_frame)(grant_ref_t ref);
144
140
 };
145
141
 
146
142
 struct unmap_refs_callback_data {
..
..
@@ -285,22 +281,6 @@
285
281
 }
286
282
 EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
287
283
 
288
-static int gnttab_query_foreign_access_v1(grant_ref_t ref)
289
-{
290
-	return gnttab_shared.v1[ref].flags & (GTF_reading|GTF_writing);
291
-}
292
-
293
-static int gnttab_query_foreign_access_v2(grant_ref_t ref)
294
-{
295
-	return grstatus[ref] & (GTF_reading|GTF_writing);
296
-}
297
-
298
-int gnttab_query_foreign_access(grant_ref_t ref)
299
-{
300
-	return gnttab_interface->query_foreign_access(ref);
301
-}
302
-EXPORT_SYMBOL_GPL(gnttab_query_foreign_access);
303
-
304
284
 static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly)
305
285
 {
306
286
 	u16 flags, nflags;
..
..
@@ -354,6 +334,16 @@
354
334
 }
355
335
 EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
356
336
 
337
+static unsigned long gnttab_read_frame_v1(grant_ref_t ref)
338
+{
339
+	return gnttab_shared.v1[ref].frame;
340
+}
341
+
342
+static unsigned long gnttab_read_frame_v2(grant_ref_t ref)
343
+{
344
+	return gnttab_shared.v2[ref].full_page.frame;
345
+}
346
+
357
347
 struct deferred_entry {
358
348
 	struct list_head list;
359
349
 	grant_ref_t ref;
..
..
@@ -383,12 +373,9 @@
383
373
 		spin_unlock_irqrestore(&gnttab_list_lock, flags);
384
374
 		if (_gnttab_end_foreign_access_ref(entry->ref, entry->ro)) {
385
375
 			put_free_entry(entry->ref);
386
-			if (entry->page) {
387
-				pr_debug("freeing g.e. %#x (pfn %#lx)\n",
388
-					 entry->ref, page_to_pfn(entry->page));
389
-				put_page(entry->page);
390
-			} else
391
-				pr_info("freeing g.e. %#x\n", entry->ref);
376
+			pr_debug("freeing g.e. %#x (pfn %#lx)\n",
377
+				 entry->ref, page_to_pfn(entry->page));
378
+			put_page(entry->page);
392
379
 			kfree(entry);
393
380
 			entry = NULL;
394
381
 		} else {
..
..
@@ -413,8 +400,17 @@
413
400
 static void gnttab_add_deferred(grant_ref_t ref, bool readonly,
414
401
 				struct page *page)
415
402
 {
416
-	struct deferred_entry *entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
403
+	struct deferred_entry *entry;
404
+	gfp_t gfp = (in_atomic() || irqs_disabled()) ? GFP_ATOMIC : GFP_KERNEL;
417
405
 	const char *what = KERN_WARNING "leaking";
406
+
407
+	entry = kmalloc(sizeof(*entry), gfp);
408
+	if (!page) {
409
+		unsigned long gfn = gnttab_interface->read_frame(ref);
410
+
411
+		page = pfn_to_page(gfn_to_pfn(gfn));
412
+		get_page(page);
413
+	}
418
414
 
419
415
 	if (entry) {
420
416
 		unsigned long flags;
..
..
@@ -436,11 +432,21 @@
436
432
 	       what, ref, page ? page_to_pfn(page) : -1);
437
433
 }
438
434
 
435
+int gnttab_try_end_foreign_access(grant_ref_t ref)
436
+{
437
+	int ret = _gnttab_end_foreign_access_ref(ref, 0);
438
+
439
+	if (ret)
440
+		put_free_entry(ref);
441
+
442
+	return ret;
443
+}
444
+EXPORT_SYMBOL_GPL(gnttab_try_end_foreign_access);
445
+
439
446
 void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
440
447
 			       unsigned long page)
441
448
 {
442
-	if (gnttab_end_foreign_access_ref(ref, readonly)) {
443
-		put_free_entry(ref);
449
+	if (gnttab_try_end_foreign_access(ref)) {
444
450
 		if (page != 0)
445
451
 			put_page(virt_to_page(page));
446
452
 	} else
..
..
@@ -664,7 +670,6 @@
664
670
 	unsigned int nr_glist_frames, new_nr_glist_frames;
665
671
 	unsigned int grefs_per_frame;
666
672
 
667
-	BUG_ON(gnttab_interface == NULL);
668
673
 	grefs_per_frame = gnttab_interface->grefs_per_grant_frame;
669
674
 
670
675
 	new_nr_grant_frames = nr_grant_frames + more_frames;
..
..
@@ -803,7 +808,7 @@
803
808
 {
804
809
 	int ret;
805
810
 
806
-	ret = alloc_xenballooned_pages(nr_pages, pages);
811
+	ret = xen_alloc_unpopulated_pages(nr_pages, pages);
807
812
 	if (ret < 0)
808
813
 		return ret;
809
814
 
..
..
@@ -814,6 +819,129 @@
814
819
 	return ret;
815
820
 }
816
821
 EXPORT_SYMBOL_GPL(gnttab_alloc_pages);
822
+
823
+#ifdef CONFIG_XEN_UNPOPULATED_ALLOC
824
+static inline void cache_init(struct gnttab_page_cache *cache)
825
+{
826
+	cache->pages = NULL;
827
+}
828
+
829
+static inline bool cache_empty(struct gnttab_page_cache *cache)
830
+{
831
+	return !cache->pages;
832
+}
833
+
834
+static inline struct page *cache_deq(struct gnttab_page_cache *cache)
835
+{
836
+	struct page *page;
837
+
838
+	page = cache->pages;
839
+	cache->pages = page->zone_device_data;
840
+
841
+	return page;
842
+}
843
+
844
+static inline void cache_enq(struct gnttab_page_cache *cache, struct page *page)
845
+{
846
+	page->zone_device_data = cache->pages;
847
+	cache->pages = page;
848
+}
849
+#else
850
+static inline void cache_init(struct gnttab_page_cache *cache)
851
+{
852
+	INIT_LIST_HEAD(&cache->pages);
853
+}
854
+
855
+static inline bool cache_empty(struct gnttab_page_cache *cache)
856
+{
857
+	return list_empty(&cache->pages);
858
+}
859
+
860
+static inline struct page *cache_deq(struct gnttab_page_cache *cache)
861
+{
862
+	struct page *page;
863
+
864
+	page = list_first_entry(&cache->pages, struct page, lru);
865
+	list_del(&page->lru);
866
+
867
+	return page;
868
+}
869
+
870
+static inline void cache_enq(struct gnttab_page_cache *cache, struct page *page)
871
+{
872
+	list_add(&page->lru, &cache->pages);
873
+}
874
+#endif
875
+
876
+void gnttab_page_cache_init(struct gnttab_page_cache *cache)
877
+{
878
+	spin_lock_init(&cache->lock);
879
+	cache_init(cache);
880
+	cache->num_pages = 0;
881
+}
882
+EXPORT_SYMBOL_GPL(gnttab_page_cache_init);
883
+
884
+int gnttab_page_cache_get(struct gnttab_page_cache *cache, struct page **page)
885
+{
886
+	unsigned long flags;
887
+
888
+	spin_lock_irqsave(&cache->lock, flags);
889
+
890
+	if (cache_empty(cache)) {
891
+		spin_unlock_irqrestore(&cache->lock, flags);
892
+		return gnttab_alloc_pages(1, page);
893
+	}
894
+
895
+	page[0] = cache_deq(cache);
896
+	cache->num_pages--;
897
+
898
+	spin_unlock_irqrestore(&cache->lock, flags);
899
+
900
+	return 0;
901
+}
902
+EXPORT_SYMBOL_GPL(gnttab_page_cache_get);
903
+
904
+void gnttab_page_cache_put(struct gnttab_page_cache *cache, struct page **page,
905
+			   unsigned int num)
906
+{
907
+	unsigned long flags;
908
+	unsigned int i;
909
+
910
+	spin_lock_irqsave(&cache->lock, flags);
911
+
912
+	for (i = 0; i < num; i++)
913
+		cache_enq(cache, page[i]);
914
+	cache->num_pages += num;
915
+
916
+	spin_unlock_irqrestore(&cache->lock, flags);
917
+}
918
+EXPORT_SYMBOL_GPL(gnttab_page_cache_put);
919
+
920
+void gnttab_page_cache_shrink(struct gnttab_page_cache *cache, unsigned int num)
921
+{
922
+	struct page *page[10];
923
+	unsigned int i = 0;
924
+	unsigned long flags;
925
+
926
+	spin_lock_irqsave(&cache->lock, flags);
927
+
928
+	while (cache->num_pages > num) {
929
+		page[i] = cache_deq(cache);
930
+		cache->num_pages--;
931
+		if (++i == ARRAY_SIZE(page)) {
932
+			spin_unlock_irqrestore(&cache->lock, flags);
933
+			gnttab_free_pages(i, page);
934
+			i = 0;
935
+			spin_lock_irqsave(&cache->lock, flags);
936
+		}
937
+	}
938
+
939
+	spin_unlock_irqrestore(&cache->lock, flags);
940
+
941
+	if (i != 0)
942
+		gnttab_free_pages(i, page);
943
+}
944
+EXPORT_SYMBOL_GPL(gnttab_page_cache_shrink);
817
945
 
818
946
 void gnttab_pages_clear_private(int nr_pages, struct page **pages)
819
947
 {
..
..
@@ -838,7 +966,7 @@
838
966
 void gnttab_free_pages(int nr_pages, struct page **pages)
839
967
 {
840
968
 	gnttab_pages_clear_private(nr_pages, pages);
841
-	free_xenballooned_pages(nr_pages, pages);
969
+	xen_free_unpopulated_pages(nr_pages, pages);
842
970
 }
843
971
 EXPORT_SYMBOL_GPL(gnttab_free_pages);
844
972
 
..
..
@@ -852,6 +980,9 @@
852
980
 	unsigned long pfn, start_pfn;
853
981
 	size_t size;
854
982
 	int i, ret;
983
+
984
+	if (args->nr_pages < 0 || args->nr_pages > (INT_MAX >> PAGE_SHIFT))
985
+		return -ENOMEM;
855
986
 
856
987
 	size = args->nr_pages << PAGE_SHIFT;
857
988
 	if (args->coherent)
..
..
@@ -1160,7 +1291,6 @@
1160
1291
 
1161
1292
 static unsigned int nr_status_frames(unsigned int nr_grant_frames)
1162
1293
 {
1163
-	BUG_ON(gnttab_interface == NULL);
1164
1294
 	return gnttab_frames(nr_grant_frames, SPP);
1165
1295
 }
1166
1296
 
..
..
@@ -1297,7 +1427,7 @@
1297
1427
 	.update_entry			= gnttab_update_entry_v1,
1298
1428
 	.end_foreign_access_ref		= gnttab_end_foreign_access_ref_v1,
1299
1429
 	.end_foreign_transfer_ref	= gnttab_end_foreign_transfer_ref_v1,
1300
-	.query_foreign_access		= gnttab_query_foreign_access_v1,
1430
+	.read_frame			= gnttab_read_frame_v1,
1301
1431
 };
1302
1432
 
1303
1433
 static const struct gnttab_ops gnttab_v2_ops = {
..
..
@@ -1309,7 +1439,7 @@
1309
1439
 	.update_entry			= gnttab_update_entry_v2,
1310
1440
 	.end_foreign_access_ref		= gnttab_end_foreign_access_ref_v2,
1311
1441
 	.end_foreign_transfer_ref	= gnttab_end_foreign_transfer_ref_v2,
1312
-	.query_foreign_access		= gnttab_query_foreign_access_v2,
1442
+	.read_frame			= gnttab_read_frame_v2,
1313
1443
 };
1314
1444
 
1315
1445
 static bool gnttab_need_v2(void)
..
..
@@ -1363,8 +1493,7 @@
1363
1493
 	if (xen_feature(XENFEAT_auto_translated_physmap) && gnttab_shared.addr == NULL) {
1364
1494
 		gnttab_shared.addr = xen_auto_xlat_grant_frames.vaddr;
1365
1495
 		if (gnttab_shared.addr == NULL) {
1366
-			pr_warn("gnttab share frames (addr=0x%08lx) is not mapped!\n",
1367
-				(unsigned long)xen_auto_xlat_grant_frames.vaddr);
1496
+			pr_warn("gnttab share frames is not mapped!\n");
1368
1497
 			return -ENOMEM;
1369
1498
 		}
1370
1499
 	}
..
..
@@ -1389,7 +1518,6 @@
1389
1518
 	int rc;
1390
1519
 	unsigned int cur, extra;
1391
1520
 
1392
-	BUG_ON(gnttab_interface == NULL);
1393
1521
 	cur = nr_grant_frames;
1394
1522
 	extra = ((req_entries + gnttab_interface->grefs_per_grant_frame - 1) /
1395
1523
 		 gnttab_interface->grefs_per_grant_frame);
..
..
@@ -1424,7 +1552,6 @@
1424
1552
 	/* Determine the maximum number of frames required for the
1425
1553
 	 * grant reference free list on the current hypervisor.
1426
1554
 	 */
1427
-	BUG_ON(gnttab_interface == NULL);
1428
1555
 	max_nr_glist_frames = (max_nr_grant_frames *
1429
1556
 			       gnttab_interface->grefs_per_grant_frame / RPP);
1430
1557