forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2024-05-10 093a6c67005148ae32a5c9e4553491b9f5c2457b 
 kernel/include/linux/gfp.h
....@@ -24,27 +24,38 @@
2424 #define ___GFP_HIGH		0x20u
2525 #define ___GFP_IO		0x40u
2626 #define ___GFP_FS		0x80u
27
-#define ___GFP_WRITE		0x100u
28
-#define ___GFP_NOWARN		0x200u
29
-#define ___GFP_RETRY_MAYFAIL	0x400u
30
-#define ___GFP_NOFAIL		0x800u
31
-#define ___GFP_NORETRY		0x1000u
32
-#define ___GFP_MEMALLOC		0x2000u
33
-#define ___GFP_COMP		0x4000u
34
-#define ___GFP_ZERO		0x8000u
35
-#define ___GFP_NOMEMALLOC	0x10000u
36
-#define ___GFP_HARDWALL		0x20000u
37
-#define ___GFP_THISNODE		0x40000u
38
-#define ___GFP_ATOMIC		0x80000u
39
-#define ___GFP_ACCOUNT		0x100000u
40
-#define ___GFP_DIRECT_RECLAIM	0x200000u
41
-#define ___GFP_KSWAPD_RECLAIM	0x400000u
27
+#define ___GFP_ZERO		0x100u
28
+#define ___GFP_ATOMIC		0x200u
29
+#define ___GFP_DIRECT_RECLAIM	0x400u
30
+#define ___GFP_KSWAPD_RECLAIM	0x800u
31
+#define ___GFP_WRITE		0x1000u
32
+#define ___GFP_NOWARN		0x2000u
33
+#define ___GFP_RETRY_MAYFAIL	0x4000u
34
+#define ___GFP_NOFAIL		0x8000u
35
+#define ___GFP_NORETRY		0x10000u
36
+#define ___GFP_MEMALLOC		0x20000u
37
+#define ___GFP_COMP		0x40000u
38
+#define ___GFP_NOMEMALLOC	0x80000u
39
+#define ___GFP_HARDWALL		0x100000u
40
+#define ___GFP_THISNODE		0x200000u
41
+#define ___GFP_ACCOUNT		0x400000u
42
+#define ___GFP_ZEROTAGS		0x800000u
43
+#define ___GFP_SKIP_KASAN_POISON	0x1000000u
44
+#ifdef CONFIG_CMA
45
+#define ___GFP_CMA		0x2000000u
46
+#else
47
+#define ___GFP_CMA		0
48
+#endif
4249 #ifdef CONFIG_LOCKDEP
43
-#define ___GFP_NOLOCKDEP	0x800000u
50
+#ifdef CONFIG_CMA
51
+#define ___GFP_NOLOCKDEP	0x4000000u
52
+#else
53
+#define ___GFP_NOLOCKDEP	0x2000000u
54
+#endif
4455 #else
4556 #define ___GFP_NOLOCKDEP	0
4657 #endif
47
-#define ___GFP_CMA		0x1000000u
58
+
4859 /* If the above are modified, __GFP_BITS_SHIFT may need updating */
4960 
5061 /*
....@@ -83,7 +94,7 @@
8394  *
8495  * %__GFP_HARDWALL enforces the cpuset memory allocation policy.
8596  *
86
- * %__GFP_THISNODE forces the allocation to be satisified from the requested
97
+ * %__GFP_THISNODE forces the allocation to be satisfied from the requested
8798  * node with no fallbacks or placement policy enforcements.
8899  *
89100  * %__GFP_ACCOUNT causes the allocation to be accounted to kmemcg.
....@@ -112,6 +123,11 @@
112123  * the caller guarantees the allocation will allow more memory to be freed
113124  * very shortly e.g. process exiting or swapping. Users either should
114125  * be the MM or co-ordinating closely with the VM (e.g. swap over NFS).
126
+ * Users of this flag have to be extremely careful to not deplete the reserve
127
+ * completely and implement a throttling mechanism which controls the
128
+ * consumption of the reserve based on the amount of freed memory.
129
+ * Usage of a pre-allocated pool (e.g. mempool) should be always considered
130
+ * before using this flag.
115131  *
116132  * %__GFP_NOMEMALLOC is used to explicitly forbid access to emergency reserves.
117133  * This takes precedence over the %__GFP_MEMALLOC flag if both are set.
....@@ -126,6 +142,8 @@
126142  *
127143  * Reclaim modifiers
128144  * ~~~~~~~~~~~~~~~~~
145
+ * Please note that all the following flags are only applicable to sleepable
146
+ * allocations (e.g. %GFP_NOWAIT and %GFP_ATOMIC will ignore them).
129147  *
130148  * %__GFP_IO can start physical IO.
131149  *
....@@ -210,22 +228,30 @@
210228  * %__GFP_COMP address compound page metadata.
211229  *
212230  * %__GFP_ZERO returns a zeroed page on success.
231
+ *
232
+ * %__GFP_ZEROTAGS returns a page with zeroed memory tags on success, if
233
+ * __GFP_ZERO is set.
234
+ *
235
+ * %__GFP_SKIP_KASAN_POISON returns a page which does not need to be poisoned
236
+ * on deallocation. Typically used for userspace pages. Currently only has an
237
+ * effect in HW tags mode.
213238  */
214239 #define __GFP_NOWARN	((__force gfp_t)___GFP_NOWARN)
215240 #define __GFP_COMP	((__force gfp_t)___GFP_COMP)
216241 #define __GFP_ZERO	((__force gfp_t)___GFP_ZERO)
242
+#define __GFP_ZEROTAGS	((__force gfp_t)___GFP_ZEROTAGS)
243
+#define __GFP_SKIP_KASAN_POISON	((__force gfp_t)___GFP_SKIP_KASAN_POISON)
217244 
218245 /* Disable lockdep for GFP context tracking */
219246 #define __GFP_NOLOCKDEP ((__force gfp_t)___GFP_NOLOCKDEP)
220247 
221248 /* Room for N __GFP_FOO bits */
222
-#define __GFP_BITS_SHIFT (25)
223
-#ifdef CONFIG_LOCKDEP
224
-#define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
249
+#ifdef CONFIG_CMA
250
+#define __GFP_BITS_SHIFT (26 + IS_ENABLED(CONFIG_LOCKDEP))
225251 #else
226
-#define __GFP_BITS_MASK (((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) & \
227
-				~0x800000u)
252
+#define __GFP_BITS_SHIFT (25 + IS_ENABLED(CONFIG_LOCKDEP))
228253 #endif
254
+#define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1))
229255 
230256 /**
231257  * DOC: Useful GFP flag combinations
....@@ -238,7 +264,9 @@
238264  * %__GFP_FOO flags as necessary.
239265  *
240266  * %GFP_ATOMIC users can not sleep and need the allocation to succeed. A lower
241
- * watermark is applied to allow access to "atomic reserves"
267
+ * watermark is applied to allow access to "atomic reserves".
268
+ * The current implementation doesn't support NMI and few other strict
269
+ * non-preemptive contexts (e.g. raw_spin_lock). The same applies to %GFP_NOWAIT.
242270  *
243271  * %GFP_KERNEL is typical for kernel-internal allocations. The caller requires
244272  * %ZONE_NORMAL or a lower zone for direct access but can direct reclaim.
....@@ -303,7 +331,8 @@
303331 #define GFP_DMA		__GFP_DMA
304332 #define GFP_DMA32	__GFP_DMA32
305333 #define GFP_HIGHUSER	(GFP_USER | __GFP_HIGHMEM)
306
-#define GFP_HIGHUSER_MOVABLE	(GFP_HIGHUSER | __GFP_MOVABLE)
334
+#define GFP_HIGHUSER_MOVABLE	(GFP_HIGHUSER | __GFP_MOVABLE | \
335
+			 __GFP_SKIP_KASAN_POISON)
307336 #define GFP_TRANSHUGE_LIGHT	((GFP_HIGHUSER_MOVABLE | __GFP_COMP | \
308337 			 __GFP_NOMEMALLOC | __GFP_NOWARN) & ~__GFP_RECLAIM)
309338 #define GFP_TRANSHUGE	(GFP_TRANSHUGE_LIGHT | __GFP_DIRECT_RECLAIM)
....@@ -312,7 +341,7 @@
312341 #define GFP_MOVABLE_MASK (__GFP_RECLAIMABLE|__GFP_MOVABLE)
313342 #define GFP_MOVABLE_SHIFT 3
314343 
315
-static inline int gfpflags_to_migratetype(const gfp_t gfp_flags)
344
+static inline int gfp_migratetype(const gfp_t gfp_flags)
316345 {
317346 	VM_WARN_ON((gfp_flags & GFP_MOVABLE_MASK) == GFP_MOVABLE_MASK);
318347 	BUILD_BUG_ON((1UL << GFP_MOVABLE_SHIFT) != ___GFP_MOVABLE);
....@@ -445,16 +474,7 @@
445474 	| 1 << (___GFP_MOVABLE | ___GFP_DMA32 | ___GFP_DMA | ___GFP_HIGHMEM)  \
446475 )
447476 
448
-static inline enum zone_type gfp_zone(gfp_t flags)
449
-{
450
-	enum zone_type z;
451
-	int bit = (__force int) (flags & GFP_ZONEMASK);
452
-
453
-	z = (GFP_ZONE_TABLE >> (bit * GFP_ZONES_SHIFT)) &
454
-					 ((1 << GFP_ZONES_SHIFT) - 1);
455
-	VM_BUG_ON((GFP_ZONE_BAD >> bit) & 1);
456
-	return z;
457
-}
477
+enum zone_type gfp_zone(gfp_t flags);
458478 
459479 /*
460480  * There is only one page-allocator function, and two main namespaces to
....@@ -491,6 +511,12 @@
491511 #endif
492512 #ifndef HAVE_ARCH_ALLOC_PAGE
493513 static inline void arch_alloc_page(struct page *page, int order) { }
514
+#endif
515
+#ifndef HAVE_ARCH_MAKE_PAGE_ACCESSIBLE
516
+static inline int arch_make_page_accessible(struct page *page)
517
+{
518
+	return 0;
519
+}
494520 #endif
495521 
496522 struct page *
....@@ -541,21 +567,21 @@
541567 extern struct page *alloc_pages_vma(gfp_t gfp_mask, int order,
542568 			struct vm_area_struct *vma, unsigned long addr,
543569 			int node, bool hugepage);
544
-#define alloc_hugepage_vma(gfp_mask, vma, addr, order)	\
570
+#define alloc_hugepage_vma(gfp_mask, vma, addr, order) \
545571 	alloc_pages_vma(gfp_mask, order, vma, addr, numa_node_id(), true)
546572 #else
547
-#define alloc_pages(gfp_mask, order) \
548
-		alloc_pages_node(numa_node_id(), gfp_mask, order)
573
+static inline struct page *alloc_pages(gfp_t gfp_mask, unsigned int order)
574
+{
575
+	return alloc_pages_node(numa_node_id(), gfp_mask, order);
576
+}
549577 #define alloc_pages_vma(gfp_mask, order, vma, addr, node, false)\
550578 	alloc_pages(gfp_mask, order)
551
-#define alloc_hugepage_vma(gfp_mask, vma, addr, order)	\
579
+#define alloc_hugepage_vma(gfp_mask, vma, addr, order) \
552580 	alloc_pages(gfp_mask, order)
553581 #endif
554582 #define alloc_page(gfp_mask) alloc_pages(gfp_mask, 0)
555583 #define alloc_page_vma(gfp_mask, vma, addr)			\
556584 	alloc_pages_vma(gfp_mask, 0, vma, addr, numa_node_id(), false)
557
-#define alloc_page_vma_node(gfp_mask, vma, addr, node)		\
558
-	alloc_pages_vma(gfp_mask, 0, vma, addr, node, false)
559585 
560586 extern unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order);
561587 extern unsigned long get_zeroed_page(gfp_t gfp_mask);
....@@ -615,12 +641,29 @@
615641 }
616642 #endif /* CONFIG_PM_SLEEP */
617643 
618
-#if (defined(CONFIG_MEMORY_ISOLATION) && defined(CONFIG_COMPACTION)) || defined(CONFIG_CMA)
644
+#ifdef CONFIG_CONTIG_ALLOC
645
+extern unsigned long pfn_max_align_up(unsigned long pfn);
646
+
647
+#define ACR_ERR_ISOLATE	(1 << 0)
648
+#define ACR_ERR_MIGRATE	(1 << 1)
649
+#define ACR_ERR_TEST	(1 << 2)
650
+
651
+struct acr_info {
652
+	unsigned long nr_mapped;
653
+	unsigned long nr_migrated;
654
+	unsigned long nr_reclaimed;
655
+	unsigned int err;
656
+	unsigned long failed_pfn;
657
+};
658
+
619659 /* The below functions must be run on a range from a single zone. */
620660 extern int alloc_contig_range(unsigned long start, unsigned long end,
621
-			      unsigned migratetype, gfp_t gfp_mask);
622
-extern void free_contig_range(unsigned long pfn, unsigned nr_pages);
661
+			      unsigned migratetype, gfp_t gfp_mask,
662
+			      struct acr_info *info);
663
+extern struct page *alloc_contig_pages(unsigned long nr_pages, gfp_t gfp_mask,
664
+				       int nid, nodemask_t *nodemask);
623665 #endif
666
+void free_contig_range(unsigned long pfn, unsigned int nr_pages);
624667 
625668 #ifdef CONFIG_CMA
626669 /* CMA stuff */