~hc/RK356X_SDK_RELEASE.git

..	..	@@ -100,6 +100,7 @@
100	100	#include <linux/seq_file.h>
101	101	#include <linux/notifier.h>
102	102	#include <linux/kallsyms.h>
	103	+#include <linux/kfence.h>
103	104	#include <linux/cpu.h>
104	105	#include <linux/sysctl.h>
105	106	#include <linux/module.h>
..	..	@@ -233,7 +234,7 @@
233	234	parent->shared = NULL;
234	235	parent->alien = NULL;
235	236	parent->colour_next = 0;
236		- raw_spin_lock_init(&parent->list_lock);
	237	+ spin_lock_init(&parent->list_lock);
237	238	parent->free_objects = 0;
238	239	parent->free_touched = 0;
239	240	}
..	..	@@ -362,29 +363,6 @@
362	363
363	364	#endif
364	365
365		-#ifdef CONFIG_DEBUG_SLAB_LEAK
366		-
367		-static inline bool is_store_user_clean(struct kmem_cache *cachep)
368		-{
369		- return atomic_read(&cachep->store_user_clean) == 1;
370		-}
371		-
372		-static inline void set_store_user_clean(struct kmem_cache *cachep)
373		-{
374		- atomic_set(&cachep->store_user_clean, 1);
375		-}
376		-
377		-static inline void set_store_user_dirty(struct kmem_cache *cachep)
378		-{
379		- if (is_store_user_clean(cachep))
380		- atomic_set(&cachep->store_user_clean, 0);
381		-}
382		-
383		-#else
384		-static inline void set_store_user_dirty(struct kmem_cache *cachep) {}
385		-
386		-#endif
387		-
388	366	/*
389	367	* Do not go above this order unless 0 objects fit into the slab or
390	368	* overridden on the command line.
..	..	@@ -393,12 +371,6 @@
393	371	#define SLAB_MAX_ORDER_LO 0
394	372	static int slab_max_order = SLAB_MAX_ORDER_LO;
395	373	static bool slab_max_order_set __initdata;
396		-
397		-static inline struct kmem_cache virt_to_cache(const void obj)
398		-{
399		- struct page *page = virt_to_head_page(obj);
400		- return page->slab_cache;
401		-}
402	374
403	375	static inline void index_to_obj(struct kmem_cache cache, struct page *page,
404	376	unsigned int idx)
..	..	@@ -587,9 +559,9 @@
587	559	page_node = page_to_nid(page);
588	560	n = get_node(cachep, page_node);
589	561
590		- raw_spin_lock(&n->list_lock);
	562	+ spin_lock(&n->list_lock);
591	563	free_block(cachep, &objp, 1, page_node, &list);
592		- raw_spin_unlock(&n->list_lock);
	564	+ spin_unlock(&n->list_lock);
593	565
594	566	slabs_destroy(cachep, &list);
595	567	}
..	..	@@ -615,6 +587,16 @@
615	587	from->avail -= nr;
616	588	to->avail += nr;
617	589	return nr;
	590	+}
	591	+
	592	+/* &alien->lock must be held by alien callers. */
	593	+static __always_inline void __free_one(struct array_cache ac, void objp)
	594	+{
	595	+ /* Avoid trivial double-free. */
	596	+ if (IS_ENABLED(CONFIG_SLAB_FREELIST_HARDENED) &&
	597	+ WARN_ON_ONCE(ac->avail > 0 && ac->entry[ac->avail - 1] == objp))
	598	+ return;
	599	+ ac->entry[ac->avail++] = objp;
618	600	}
619	601
620	602	#ifndef CONFIG_NUMA
..	..	@@ -677,12 +659,11 @@
677	659	static struct alien_cache **alloc_alien_cache(int node, int limit, gfp_t gfp)
678	660	{
679	661	struct alien_cache **alc_ptr;
680		- size_t memsize = sizeof(void ) nr_node_ids;
681	662	int i;
682	663
683	664	if (limit > 1)
684	665	limit = 12;
685		- alc_ptr = kzalloc_node(memsize, gfp, node);
	666	+ alc_ptr = kcalloc_node(nr_node_ids, sizeof(void *), gfp, node);
686	667	if (!alc_ptr)
687	668	return NULL;
688	669
..	..	@@ -718,7 +699,7 @@
718	699	struct kmem_cache_node *n = get_node(cachep, node);
719	700
720	701	if (ac->avail) {
721		- raw_spin_lock(&n->list_lock);
	702	+ spin_lock(&n->list_lock);
722	703	/*
723	704	* Stuff objects into the remote nodes shared array first.
724	705	* That way we could avoid the overhead of putting the objects
..	..	@@ -729,7 +710,7 @@
729	710
730	711	free_block(cachep, ac->entry, ac->avail, node, list);
731	712	ac->avail = 0;
732		- raw_spin_unlock(&n->list_lock);
	713	+ spin_unlock(&n->list_lock);
733	714	}
734	715	}
735	716
..	..	@@ -797,14 +778,14 @@
797	778	STATS_INC_ACOVERFLOW(cachep);
798	779	__drain_alien_cache(cachep, ac, page_node, &list);
799	780	}
800		- ac->entry[ac->avail++] = objp;
	781	+ __free_one(ac, objp);
801	782	spin_unlock(&alien->lock);
802	783	slabs_destroy(cachep, &list);
803	784	} else {
804	785	n = get_node(cachep, page_node);
805		- raw_spin_lock(&n->list_lock);
	786	+ spin_lock(&n->list_lock);
806	787	free_block(cachep, &objp, 1, page_node, &list);
807		- raw_spin_unlock(&n->list_lock);
	788	+ spin_unlock(&n->list_lock);
808	789	slabs_destroy(cachep, &list);
809	790	}
810	791	return 1;
..	..	@@ -845,10 +826,10 @@
845	826	*/
846	827	n = get_node(cachep, node);
847	828	if (n) {
848		- raw_spin_lock_irq(&n->list_lock);
	829	+ spin_lock_irq(&n->list_lock);
849	830	n->free_limit = (1 + nr_cpus_node(node)) * cachep->batchcount +
850	831	cachep->num;
851		- raw_spin_unlock_irq(&n->list_lock);
	832	+ spin_unlock_irq(&n->list_lock);
852	833
853	834	return 0;
854	835	}
..	..	@@ -927,7 +908,7 @@
927	908	goto fail;
928	909
929	910	n = get_node(cachep, node);
930		- raw_spin_lock_irq(&n->list_lock);
	911	+ spin_lock_irq(&n->list_lock);
931	912	if (n->shared && force_change) {
932	913	free_block(cachep, n->shared->entry,
933	914	n->shared->avail, node, &list);
..	..	@@ -945,17 +926,17 @@
945	926	new_alien = NULL;
946	927	}
947	928
948		- raw_spin_unlock_irq(&n->list_lock);
	929	+ spin_unlock_irq(&n->list_lock);
949	930	slabs_destroy(cachep, &list);
950	931
951	932	/*
952	933	* To protect lockless access to n->shared during irq disabled context.
953	934	* If n->shared isn't NULL in irq disabled context, accessing to it is
954	935	* guaranteed to be valid until irq is re-enabled, because it will be
955		- * freed after synchronize_sched().
	936	+ * freed after synchronize_rcu().
956	937	*/
957	938	if (old_shared && force_change)
958		- synchronize_sched();
	939	+ synchronize_rcu();
959	940
960	941	fail:
961	942	kfree(old_shared);
..	..	@@ -984,20 +965,18 @@
984	965	if (!n)
985	966	continue;
986	967
987		- raw_spin_lock_irq(&n->list_lock);
	968	+ spin_lock_irq(&n->list_lock);
988	969
989	970	/* Free limit for this kmem_cache_node */
990	971	n->free_limit -= cachep->batchcount;
991	972
992	973	/* cpu is dead; no one can alloc from it. */
993	974	nc = per_cpu_ptr(cachep->cpu_cache, cpu);
994		- if (nc) {
995		- free_block(cachep, nc->entry, nc->avail, node, &list);
996		- nc->avail = 0;
997		- }
	975	+ free_block(cachep, nc->entry, nc->avail, node, &list);
	976	+ nc->avail = 0;
998	977
999	978	if (!cpumask_empty(mask)) {
1000		- raw_spin_unlock_irq(&n->list_lock);
	979	+ spin_unlock_irq(&n->list_lock);
1001	980	goto free_slab;
1002	981	}
1003	982
..	..	@@ -1011,7 +990,7 @@
1011	990	alien = n->alien;
1012	991	n->alien = NULL;
1013	992
1014		- raw_spin_unlock_irq(&n->list_lock);
	993	+ spin_unlock_irq(&n->list_lock);
1015	994
1016	995	kfree(shared);
1017	996	if (alien) {
..	..	@@ -1082,9 +1061,9 @@
1082	1061	* offline.
1083	1062	*
1084	1063	* Even if all the cpus of a node are down, we don't free the
1085		- * kmem_list3 of any cache. This to avoid a race between cpu_down, and
	1064	+ * kmem_cache_node of any cache. This to avoid a race between cpu_down, and
1086	1065	* a kmalloc allocation from another cpu for memory from the node of
1087		- * the cpu going down. The list3 structure is usually allocated from
	1066	+ * the cpu going down. The kmem_cache_node structure is usually allocated from
1088	1067	* kmem_cache_create() and gets destroyed at kmem_cache_destroy().
1089	1068	*/
1090	1069	int slab_dead_cpu(unsigned int cpu)
..	..	@@ -1195,7 +1174,7 @@
1195	1174	/*
1196	1175	* Do not assume that spinlocks can be initialized via memcpy:
1197	1176	*/
1198		- raw_spin_lock_init(&ptr->list_lock);
	1177	+ spin_lock_init(&ptr->list_lock);
1199	1178
1200	1179	MAKE_ALL_LISTS(cachep, ptr, nodeid);
1201	1180	cachep->node[nodeid] = ptr;
..	..	@@ -1238,7 +1217,7 @@
1238	1217	* page orders on machines with more than 32MB of memory if
1239	1218	* not overridden on the command line.
1240	1219	*/
1241		- if (!slab_max_order_set && totalram_pages > (32 << 20) >> PAGE_SHIFT)
	1220	+ if (!slab_max_order_set && totalram_pages() > (32 << 20) >> PAGE_SHIFT)
1242	1221	slab_max_order = SLAB_MAX_ORDER_HI;
1243	1222
1244	1223	/* Bootstrap is tricky, because several objects are allocated
..	..	@@ -1271,7 +1250,6 @@
1271	1250	nr_node_ids * sizeof(struct kmem_cache_node *),
1272	1251	SLAB_HWCACHE_ALIGN, 0, 0);
1273	1252	list_add(&kmem_cache->list, &slab_caches);
1274		- memcg_link_cache(kmem_cache);
1275	1253	slab_state = PARTIAL;
1276	1254
1277	1255	/*
..	..	@@ -1279,9 +1257,10 @@
1279	1257	* structures first. Without this, further allocations will bug.
1280	1258	*/
1281	1259	kmalloc_caches[KMALLOC_NORMAL][INDEX_NODE] = create_kmalloc_cache(
1282		- kmalloc_info[INDEX_NODE].name,
1283		- kmalloc_size(INDEX_NODE), ARCH_KMALLOC_FLAGS,
1284		- 0, kmalloc_size(INDEX_NODE));
	1260	+ kmalloc_info[INDEX_NODE].name[KMALLOC_NORMAL],
	1261	+ kmalloc_info[INDEX_NODE].size,
	1262	+ ARCH_KMALLOC_FLAGS, 0,
	1263	+ kmalloc_info[INDEX_NODE].size);
1285	1264	slab_state = PARTIAL_NODE;
1286	1265	setup_kmalloc_cache_index_table();
1287	1266
..	..	@@ -1366,11 +1345,11 @@
1366	1345	for_each_kmem_cache_node(cachep, node, n) {
1367	1346	unsigned long total_slabs, free_slabs, free_objs;
1368	1347
1369		- raw_spin_lock_irqsave(&n->list_lock, flags);
	1348	+ spin_lock_irqsave(&n->list_lock, flags);
1370	1349	total_slabs = n->total_slabs;
1371	1350	free_slabs = n->free_slabs;
1372	1351	free_objs = n->free_objects;
1373		- raw_spin_unlock_irqrestore(&n->list_lock, flags);
	1352	+ spin_unlock_irqrestore(&n->list_lock, flags);
1374	1353
1375	1354	pr_warn(" node %d: slabs: %ld/%ld, objs: %ld/%ld\n",
1376	1355	node, total_slabs - free_slabs, total_slabs,
..	..	@@ -1392,7 +1371,6 @@
1392	1371	int nodeid)
1393	1372	{
1394	1373	struct page *page;
1395		- int nr_pages;
1396	1374
1397	1375	flags \|= cachep->allocflags;
1398	1376
..	..	@@ -1402,17 +1380,7 @@
1402	1380	return NULL;
1403	1381	}
1404	1382
1405		- if (memcg_charge_slab(page, flags, cachep->gfporder, cachep)) {
1406		- __free_pages(page, cachep->gfporder);
1407		- return NULL;
1408		- }
1409		-
1410		- nr_pages = (1 << cachep->gfporder);
1411		- if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1412		- mod_lruvec_page_state(page, NR_SLAB_RECLAIMABLE, nr_pages);
1413		- else
1414		- mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE, nr_pages);
1415		-
	1383	+ account_slab_page(page, cachep->gfporder, cachep);
1416	1384	__SetPageSlab(page);
1417	1385	/* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */
1418	1386	if (sk_memalloc_socks() && page_is_pfmemalloc(page))
..	..	@@ -1427,12 +1395,6 @@
1427	1395	static void kmem_freepages(struct kmem_cache cachep, struct page page)
1428	1396	{
1429	1397	int order = cachep->gfporder;
1430		- unsigned long nr_freed = (1 << order);
1431		-
1432		- if (cachep->flags & SLAB_RECLAIM_ACCOUNT)
1433		- mod_lruvec_page_state(page, NR_SLAB_RECLAIMABLE, -nr_freed);
1434		- else
1435		- mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE, -nr_freed);
1436	1398
1437	1399	BUG_ON(!PageSlab(page));
1438	1400	__ClearPageSlabPfmemalloc(page);
..	..	@@ -1441,8 +1403,8 @@
1441	1403	page->mapping = NULL;
1442	1404
1443	1405	if (current->reclaim_state)
1444		- current->reclaim_state->reclaimed_slab += nr_freed;
1445		- memcg_uncharge_slab(page, order, cachep);
	1406	+ current->reclaim_state->reclaimed_slab += 1 << order;
	1407	+ unaccount_slab_page(page, order, cachep);
1446	1408	__free_pages(page, order);
1447	1409	}
1448	1410
..	..	@@ -1460,7 +1422,7 @@
1460	1422	#if DEBUG
1461	1423	static bool is_debug_pagealloc_cache(struct kmem_cache *cachep)
1462	1424	{
1463		- if (debug_pagealloc_enabled() && OFF_SLAB(cachep) &&
	1425	+ if (debug_pagealloc_enabled_static() && OFF_SLAB(cachep) &&
1464	1426	(cachep->size % PAGE_SIZE) == 0)
1465	1427	return true;
1466	1428
..	..	@@ -1468,53 +1430,17 @@
1468	1430	}
1469	1431
1470	1432	#ifdef CONFIG_DEBUG_PAGEALLOC
1471		-static void store_stackinfo(struct kmem_cache cachep, unsigned long addr,
1472		- unsigned long caller)
1473		-{
1474		- int size = cachep->object_size;
1475		-
1476		- addr = (unsigned long )&((char )addr)[obj_offset(cachep)];
1477		-
1478		- if (size < 5 * sizeof(unsigned long))
1479		- return;
1480		-
1481		- *addr++ = 0x12345678;
1482		- *addr++ = caller;
1483		- *addr++ = smp_processor_id();
1484		- size -= 3 * sizeof(unsigned long);
1485		- {
1486		- unsigned long *sptr = &caller;
1487		- unsigned long svalue;
1488		-
1489		- while (!kstack_end(sptr)) {
1490		- svalue = *sptr++;
1491		- if (kernel_text_address(svalue)) {
1492		- *addr++ = svalue;
1493		- size -= sizeof(unsigned long);
1494		- if (size <= sizeof(unsigned long))
1495		- break;
1496		- }
1497		- }
1498		-
1499		- }
1500		- *addr++ = 0x87654321;
1501		-}
1502		-
1503		-static void slab_kernel_map(struct kmem_cache cachep, void objp,
1504		- int map, unsigned long caller)
	1433	+static void slab_kernel_map(struct kmem_cache cachep, void objp, int map)
1505	1434	{
1506	1435	if (!is_debug_pagealloc_cache(cachep))
1507	1436	return;
1508	1437
1509		- if (caller)
1510		- store_stackinfo(cachep, objp, caller);
1511		-
1512		- kernel_map_pages(virt_to_page(objp), cachep->size / PAGE_SIZE, map);
	1438	+ __kernel_map_pages(virt_to_page(objp), cachep->size / PAGE_SIZE, map);
1513	1439	}
1514	1440
1515	1441	#else
1516	1442	static inline void slab_kernel_map(struct kmem_cache cachep, void objp,
1517		- int map, unsigned long caller) {}
	1443	+ int map) {}
1518	1444
1519	1445	#endif
1520	1446
..	..	@@ -1662,7 +1588,7 @@
1662	1588
1663	1589	if (cachep->flags & SLAB_POISON) {
1664	1590	check_poison_obj(cachep, objp);
1665		- slab_kernel_map(cachep, objp, 1, 0);
	1591	+ slab_kernel_map(cachep, objp, 1);
1666	1592	}
1667	1593	if (cachep->flags & SLAB_RED_ZONE) {
1668	1594	if (*dbg_redzone1(cachep, objp) != RED_INACTIVE)
..	..	@@ -1707,12 +1633,16 @@
1707	1633	kmem_cache_free(cachep->freelist_cache, freelist);
1708	1634	}
1709	1635
	1636	+/*
	1637	+ * Update the size of the caches before calling slabs_destroy as it may
	1638	+ * recursively call kfree.
	1639	+ */
1710	1640	static void slabs_destroy(struct kmem_cache cachep, struct list_head list)
1711	1641	{
1712	1642	struct page page, n;
1713	1643
1714		- list_for_each_entry_safe(page, n, list, lru) {
1715		- list_del(&page->lru);
	1644	+ list_for_each_entry_safe(page, n, list, slab_list) {
	1645	+ list_del(&page->slab_list);
1716	1646	slab_destroy(cachep, page);
1717	1647	}
1718	1648	}
..	..	@@ -1728,6 +1658,8 @@
1728	1658	* This could be made much more intelligent. For now, try to avoid using
1729	1659	* high order pages for slabs. When the gfp() functions are more friendly
1730	1660	* towards high-order requests, this should be changed.
	1661	+ *
	1662	+ * Return: number of left-over bytes in a slab
1731	1663	*/
1732	1664	static size_t calculate_slab_order(struct kmem_cache *cachep,
1733	1665	size_t size, slab_flags_t flags)
..	..	@@ -1858,8 +1790,7 @@
1858	1790	}
1859	1791
1860	1792	slab_flags_t kmem_cache_flags(unsigned int object_size,
1861		- slab_flags_t flags, const char *name,
1862		- void (ctor)(void ))
	1793	+ slab_flags_t flags, const char *name)
1863	1794	{
1864	1795	return flags;
1865	1796	}
..	..	@@ -1984,6 +1915,8 @@
1984	1915	* %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware
1985	1916	* cacheline. This can be beneficial if you're counting cycles as closely
1986	1917	* as davem.
	1918	+ *
	1919	+ * Return: a pointer to the created cache or %NULL in case of error
1987	1920	*/
1988	1921	int __kmem_cache_create(struct kmem_cache *cachep, slab_flags_t flags)
1989	1922	{
..	..	@@ -2084,7 +2017,7 @@
2084	2017	* to check size >= 256. It guarantees that all necessary small
2085	2018	* sized slab is initialized in current slab initialization sequence.
2086	2019	*/
2087		- if (debug_pagealloc_enabled() && (flags & SLAB_POISON) &&
	2020	+ if (debug_pagealloc_enabled_static() && (flags & SLAB_POISON) &&
2088	2021	size >= 256 && cachep->object_size > cache_line_size()) {
2089	2022	if (size < PAGE_SIZE \|\| size % PAGE_SIZE == 0) {
2090	2023	size_t tmp_size = ALIGN(size, PAGE_SIZE);
..	..	@@ -2173,7 +2106,7 @@
2173	2106	{
2174	2107	#ifdef CONFIG_SMP
2175	2108	check_irq_off();
2176		- assert_raw_spin_locked(&get_node(cachep, numa_mem_id())->list_lock);
	2109	+ assert_spin_locked(&get_node(cachep, numa_mem_id())->list_lock);
2177	2110	#endif
2178	2111	}
2179	2112
..	..	@@ -2181,7 +2114,7 @@
2181	2114	{
2182	2115	#ifdef CONFIG_SMP
2183	2116	check_irq_off();
2184		- assert_raw_spin_locked(&get_node(cachep, node)->list_lock);
	2117	+ assert_spin_locked(&get_node(cachep, node)->list_lock);
2185	2118	#endif
2186	2119	}
2187	2120
..	..	@@ -2221,11 +2154,11 @@
2221	2154	check_irq_off();
2222	2155	ac = cpu_cache_get(cachep);
2223	2156	n = get_node(cachep, node);
2224		- raw_spin_lock(&n->list_lock);
	2157	+ spin_lock(&n->list_lock);
2225	2158	free_block(cachep, ac->entry, ac->avail, node, &list);
2226		- raw_spin_unlock(&n->list_lock);
2227		- slabs_destroy(cachep, &list);
	2159	+ spin_unlock(&n->list_lock);
2228	2160	ac->avail = 0;
	2161	+ slabs_destroy(cachep, &list);
2229	2162	}
2230	2163
2231	2164	static void drain_cpu_caches(struct kmem_cache *cachep)
..	..	@@ -2241,9 +2174,9 @@
2241	2174	drain_alien_cache(cachep, n->alien);
2242	2175
2243	2176	for_each_kmem_cache_node(cachep, node, n) {
2244		- raw_spin_lock_irq(&n->list_lock);
	2177	+ spin_lock_irq(&n->list_lock);
2245	2178	drain_array_locked(cachep, n->shared, node, true, &list);
2246		- raw_spin_unlock_irq(&n->list_lock);
	2179	+ spin_unlock_irq(&n->list_lock);
2247	2180
2248	2181	slabs_destroy(cachep, &list);
2249	2182	}
..	..	@@ -2265,15 +2198,15 @@
2265	2198	nr_freed = 0;
2266	2199	while (nr_freed < tofree && !list_empty(&n->slabs_free)) {
2267	2200
2268		- raw_spin_lock_irq(&n->list_lock);
	2201	+ spin_lock_irq(&n->list_lock);
2269	2202	p = n->slabs_free.prev;
2270	2203	if (p == &n->slabs_free) {
2271		- raw_spin_unlock_irq(&n->list_lock);
	2204	+ spin_unlock_irq(&n->list_lock);
2272	2205	goto out;
2273	2206	}
2274	2207
2275		- page = list_entry(p, struct page, lru);
2276		- list_del(&page->lru);
	2208	+ page = list_entry(p, struct page, slab_list);
	2209	+ list_del(&page->slab_list);
2277	2210	n->free_slabs--;
2278	2211	n->total_slabs--;
2279	2212	/*
..	..	@@ -2281,7 +2214,7 @@
2281	2214	* to the cache.
2282	2215	*/
2283	2216	n->free_objects -= cache->num;
2284		- raw_spin_unlock_irq(&n->list_lock);
	2217	+ spin_unlock_irq(&n->list_lock);
2285	2218	slab_destroy(cache, page);
2286	2219	nr_freed++;
2287	2220	}
..	..	@@ -2318,13 +2251,6 @@
2318	2251	}
2319	2252	return (ret ? 1 : 0);
2320	2253	}
2321		-
2322		-#ifdef CONFIG_MEMCG
2323		-void __kmemcg_cache_deactivate(struct kmem_cache *cachep)
2324		-{
2325		- __kmem_cache_shrink(cachep);
2326		-}
2327		-#endif
2328	2254
2329	2255	int __kmem_cache_shutdown(struct kmem_cache *cachep)
2330	2256	{
..	..	@@ -2379,8 +2305,6 @@
2379	2305	/* Slab management obj is off-slab. */
2380	2306	freelist = kmem_cache_alloc_node(cachep->freelist_cache,
2381	2307	local_flags, nodeid);
2382		- if (!freelist)
2383		- return NULL;
2384	2308	} else {
2385	2309	/* We will use last bytes at the slab for freelist */
2386	2310	freelist = addr + (PAGE_SIZE << cachep->gfporder) -
..	..	@@ -2438,7 +2362,7 @@
2438	2362	/* need to poison the objs? */
2439	2363	if (cachep->flags & SLAB_POISON) {
2440	2364	poison_obj(cachep, objp, POISON_FREE);
2441		- slab_kernel_map(cachep, objp, 0, 0);
	2365	+ slab_kernel_map(cachep, objp, 0);
2442	2366	}
2443	2367	}
2444	2368	#endif
..	..	@@ -2595,11 +2519,6 @@
2595	2519	objp = index_to_obj(cachep, page, get_free_obj(page, page->active));
2596	2520	page->active++;
2597	2521
2598		-#if DEBUG
2599		- if (cachep->flags & SLAB_STORE_USER)
2600		- set_store_user_dirty(cachep);
2601		-#endif
2602		-
2603	2522	return objp;
2604	2523	}
2605	2524
..	..	@@ -2656,13 +2575,9 @@
2656	2575	* Be lazy and only check for valid flags here, keeping it out of the
2657	2576	* critical path in kmem_cache_alloc().
2658	2577	*/
2659		- if (unlikely(flags & GFP_SLAB_BUG_MASK)) {
2660		- gfp_t invalid_mask = flags & GFP_SLAB_BUG_MASK;
2661		- flags &= ~GFP_SLAB_BUG_MASK;
2662		- pr_warn("Unexpected gfp: %#x (%pGg). Fixing up to gfp: %#x (%pGg). Fix your code!\n",
2663		- invalid_mask, &invalid_mask, flags, &flags);
2664		- dump_stack();
2665		- }
	2578	+ if (unlikely(flags & GFP_SLAB_BUG_MASK))
	2579	+ flags = kmalloc_fix_flags(flags);
	2580	+
2666	2581	WARN_ON_ONCE(cachep->ctor && (flags & __GFP_ZERO));
2667	2582	local_flags = flags & (GFP_CONSTRAINT_MASK\|GFP_RECLAIM_MASK);
2668	2583
..	..	@@ -2732,20 +2647,20 @@
2732	2647	if (!page)
2733	2648	return;
2734	2649
2735		- INIT_LIST_HEAD(&page->lru);
	2650	+ INIT_LIST_HEAD(&page->slab_list);
2736	2651	n = get_node(cachep, page_to_nid(page));
2737	2652
2738		- raw_spin_lock(&n->list_lock);
	2653	+ spin_lock(&n->list_lock);
2739	2654	n->total_slabs++;
2740	2655	if (!page->active) {
2741		- list_add_tail(&page->lru, &(n->slabs_free));
	2656	+ list_add_tail(&page->slab_list, &n->slabs_free);
2742	2657	n->free_slabs++;
2743	2658	} else
2744	2659	fixup_slab_list(cachep, n, page, &list);
2745	2660
2746	2661	STATS_INC_GROWN(cachep);
2747	2662	n->free_objects += cachep->num - page->active;
2748		- raw_spin_unlock(&n->list_lock);
	2663	+ spin_unlock(&n->list_lock);
2749	2664
2750	2665	fixup_objfreelist_debug(cachep, &list);
2751	2666	}
..	..	@@ -2805,10 +2720,8 @@
2805	2720	*dbg_redzone1(cachep, objp) = RED_INACTIVE;
2806	2721	*dbg_redzone2(cachep, objp) = RED_INACTIVE;
2807	2722	}
2808		- if (cachep->flags & SLAB_STORE_USER) {
2809		- set_store_user_dirty(cachep);
	2723	+ if (cachep->flags & SLAB_STORE_USER)
2810	2724	dbg_userword(cachep, objp) = (void )caller;
2811		- }
2812	2725
2813	2726	objnr = obj_to_index(cachep, page, objp);
2814	2727
..	..	@@ -2817,7 +2730,7 @@
2817	2730
2818	2731	if (cachep->flags & SLAB_POISON) {
2819	2732	poison_obj(cachep, objp, POISON_FREE);
2820		- slab_kernel_map(cachep, objp, 0, caller);
	2733	+ slab_kernel_map(cachep, objp, 0);
2821	2734	}
2822	2735	return objp;
2823	2736	}
..	..	@@ -2847,9 +2760,9 @@
2847	2760	void **list)
2848	2761	{
2849	2762	/* move slabp to correct slabp list: */
2850		- list_del(&page->lru);
	2763	+ list_del(&page->slab_list);
2851	2764	if (page->active == cachep->num) {
2852		- list_add(&page->lru, &n->slabs_full);
	2765	+ list_add(&page->slab_list, &n->slabs_full);
2853	2766	if (OBJFREELIST_SLAB(cachep)) {
2854	2767	#if DEBUG
2855	2768	/* Poisoning will be done without holding the lock */
..	..	@@ -2863,7 +2776,7 @@
2863	2776	page->freelist = NULL;
2864	2777	}
2865	2778	} else
2866		- list_add(&page->lru, &n->slabs_partial);
	2779	+ list_add(&page->slab_list, &n->slabs_partial);
2867	2780	}
2868	2781
2869	2782	/* Try to find non-pfmemalloc slab if needed */
..	..	@@ -2886,20 +2799,20 @@
2886	2799	}
2887	2800
2888	2801	/* Move pfmemalloc slab to the end of list to speed up next search */
2889		- list_del(&page->lru);
	2802	+ list_del(&page->slab_list);
2890	2803	if (!page->active) {
2891		- list_add_tail(&page->lru, &n->slabs_free);
	2804	+ list_add_tail(&page->slab_list, &n->slabs_free);
2892	2805	n->free_slabs++;
2893	2806	} else
2894		- list_add_tail(&page->lru, &n->slabs_partial);
	2807	+ list_add_tail(&page->slab_list, &n->slabs_partial);
2895	2808
2896		- list_for_each_entry(page, &n->slabs_partial, lru) {
	2809	+ list_for_each_entry(page, &n->slabs_partial, slab_list) {
2897	2810	if (!PageSlabPfmemalloc(page))
2898	2811	return page;
2899	2812	}
2900	2813
2901	2814	n->free_touched = 1;
2902		- list_for_each_entry(page, &n->slabs_free, lru) {
	2815	+ list_for_each_entry(page, &n->slabs_free, slab_list) {
2903	2816	if (!PageSlabPfmemalloc(page)) {
2904	2817	n->free_slabs--;
2905	2818	return page;
..	..	@@ -2913,12 +2826,13 @@
2913	2826	{
2914	2827	struct page *page;
2915	2828
2916		- assert_raw_spin_locked(&n->list_lock);
2917		- page = list_first_entry_or_null(&n->slabs_partial, struct page, lru);
	2829	+ assert_spin_locked(&n->list_lock);
	2830	+ page = list_first_entry_or_null(&n->slabs_partial, struct page,
	2831	+ slab_list);
2918	2832	if (!page) {
2919	2833	n->free_touched = 1;
2920	2834	page = list_first_entry_or_null(&n->slabs_free, struct page,
2921		- lru);
	2835	+ slab_list);
2922	2836	if (page)
2923	2837	n->free_slabs--;
2924	2838	}
..	..	@@ -2939,10 +2853,10 @@
2939	2853	if (!gfp_pfmemalloc_allowed(flags))
2940	2854	return NULL;
2941	2855
2942		- raw_spin_lock(&n->list_lock);
	2856	+ spin_lock(&n->list_lock);
2943	2857	page = get_first_slab(n, true);
2944	2858	if (!page) {
2945		- raw_spin_unlock(&n->list_lock);
	2859	+ spin_unlock(&n->list_lock);
2946	2860	return NULL;
2947	2861	}
2948	2862
..	..	@@ -2951,7 +2865,7 @@
2951	2865
2952	2866	fixup_slab_list(cachep, n, page, &list);
2953	2867
2954		- raw_spin_unlock(&n->list_lock);
	2868	+ spin_unlock(&n->list_lock);
2955	2869	fixup_objfreelist_debug(cachep, &list);
2956	2870
2957	2871	return obj;
..	..	@@ -3010,7 +2924,7 @@
3010	2924	if (!n->free_objects && (!shared \|\| !shared->avail))
3011	2925	goto direct_grow;
3012	2926
3013		- raw_spin_lock(&n->list_lock);
	2927	+ spin_lock(&n->list_lock);
3014	2928	shared = READ_ONCE(n->shared);
3015	2929
3016	2930	/* See if we can refill from the shared array */
..	..	@@ -3034,7 +2948,7 @@
3034	2948	must_grow:
3035	2949	n->free_objects -= ac->avail;
3036	2950	alloc_done:
3037		- raw_spin_unlock(&n->list_lock);
	2951	+ spin_unlock(&n->list_lock);
3038	2952	fixup_objfreelist_debug(cachep, &list);
3039	2953
3040	2954	direct_grow:
..	..	@@ -3081,7 +2995,7 @@
3081	2995	return objp;
3082	2996	if (cachep->flags & SLAB_POISON) {
3083	2997	check_poison_obj(cachep, objp);
3084		- slab_kernel_map(cachep, objp, 1, 0);
	2998	+ slab_kernel_map(cachep, objp, 1);
3085	2999	poison_obj(cachep, objp, POISON_INUSE);
3086	3000	}
3087	3001	if (cachep->flags & SLAB_STORE_USER)
..	..	@@ -3102,10 +3016,9 @@
3102	3016	objp += obj_offset(cachep);
3103	3017	if (cachep->ctor && cachep->flags & SLAB_POISON)
3104	3018	cachep->ctor(objp);
3105		- if (ARCH_SLAB_MINALIGN &&
3106		- ((unsigned long)objp & (ARCH_SLAB_MINALIGN-1))) {
3107		- pr_err("0x%px: not aligned to ARCH_SLAB_MINALIGN=%d\n",
3108		- objp, (int)ARCH_SLAB_MINALIGN);
	3019	+ if ((unsigned long)objp & (arch_slab_minalign() - 1)) {
	3020	+ pr_err("0x%px: not aligned to arch_slab_minalign()=%u\n", objp,
	3021	+ arch_slab_minalign());
3109	3022	}
3110	3023	return objp;
3111	3024	}
..	..	@@ -3184,7 +3097,7 @@
3184	3097	struct zonelist *zonelist;
3185	3098	struct zoneref *z;
3186	3099	struct zone *zone;
3187		- enum zone_type high_zoneidx = gfp_zone(flags);
	3100	+ enum zone_type highest_zoneidx = gfp_zone(flags);
3188	3101	void *obj = NULL;
3189	3102	struct page *page;
3190	3103	int nid;
..	..	@@ -3202,7 +3115,7 @@
3202	3115	* Look through allowed nodes for objects available
3203	3116	* from existing per node queues.
3204	3117	*/
3205		- for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
	3118	+ for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) {
3206	3119	nid = zone_to_nid(zone);
3207	3120
3208	3121	if (cpuset_zone_allowed(zone, flags) &&
..	..	@@ -3259,7 +3172,7 @@
3259	3172	BUG_ON(!n);
3260	3173
3261	3174	check_irq_off();
3262		- raw_spin_lock(&n->list_lock);
	3175	+ spin_lock(&n->list_lock);
3263	3176	page = get_first_slab(n, false);
3264	3177	if (!page)
3265	3178	goto must_grow;
..	..	@@ -3277,12 +3190,12 @@
3277	3190
3278	3191	fixup_slab_list(cachep, n, page, &list);
3279	3192
3280		- raw_spin_unlock(&n->list_lock);
	3193	+ spin_unlock(&n->list_lock);
3281	3194	fixup_objfreelist_debug(cachep, &list);
3282	3195	return obj;
3283	3196
3284	3197	must_grow:
3285		- raw_spin_unlock(&n->list_lock);
	3198	+ spin_unlock(&n->list_lock);
3286	3199	page = cache_grow_begin(cachep, gfp_exact_node(flags), nodeid);
3287	3200	if (page) {
3288	3201	/* This slab isn't counted yet so don't update free_objects */
..	..	@@ -3294,17 +3207,23 @@
3294	3207	}
3295	3208
3296	3209	static __always_inline void *
3297		-slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
	3210	+slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, size_t orig_size,
3298	3211	unsigned long caller)
3299	3212	{
3300	3213	unsigned long save_flags;
3301	3214	void *ptr;
3302	3215	int slab_node = numa_mem_id();
	3216	+ struct obj_cgroup *objcg = NULL;
	3217	+ bool init = false;
3303	3218
3304	3219	flags &= gfp_allowed_mask;
3305		- cachep = slab_pre_alloc_hook(cachep, flags);
	3220	+ cachep = slab_pre_alloc_hook(cachep, &objcg, 1, flags);
3306	3221	if (unlikely(!cachep))
3307	3222	return NULL;
	3223	+
	3224	+ ptr = kfence_alloc(cachep, orig_size, flags);
	3225	+ if (unlikely(ptr))
	3226	+ goto out_hooks;
3308	3227
3309	3228	cache_alloc_debugcheck_before(cachep, flags);
3310	3229	local_irq_save(save_flags);
..	..	@@ -3334,11 +3253,10 @@
3334	3253	out:
3335	3254	local_irq_restore(save_flags);
3336	3255	ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller);
	3256	+ init = slab_want_init_on_alloc(flags, cachep);
3337	3257
3338		- if (unlikely(slab_want_init_on_alloc(flags, cachep)) && ptr)
3339		- memset(ptr, 0, cachep->object_size);
3340		-
3341		- slab_post_alloc_hook(cachep, flags, 1, &ptr);
	3258	+out_hooks:
	3259	+ slab_post_alloc_hook(cachep, objcg, flags, 1, &ptr, init);
3342	3260	return ptr;
3343	3261	}
3344	3262
..	..	@@ -3375,15 +3293,21 @@
3375	3293	#endif /* CONFIG_NUMA */
3376	3294
3377	3295	static __always_inline void *
3378		-slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
	3296	+slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned long caller)
3379	3297	{
3380	3298	unsigned long save_flags;
3381	3299	void *objp;
	3300	+ struct obj_cgroup *objcg = NULL;
	3301	+ bool init = false;
3382	3302
3383	3303	flags &= gfp_allowed_mask;
3384		- cachep = slab_pre_alloc_hook(cachep, flags);
	3304	+ cachep = slab_pre_alloc_hook(cachep, &objcg, 1, flags);
3385	3305	if (unlikely(!cachep))
3386	3306	return NULL;
	3307	+
	3308	+ objp = kfence_alloc(cachep, orig_size, flags);
	3309	+ if (unlikely(objp))
	3310	+ goto out;
3387	3311
3388	3312	cache_alloc_debugcheck_before(cachep, flags);
3389	3313	local_irq_save(save_flags);
..	..	@@ -3391,11 +3315,10 @@
3391	3315	local_irq_restore(save_flags);
3392	3316	objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);
3393	3317	prefetchw(objp);
	3318	+ init = slab_want_init_on_alloc(flags, cachep);
3394	3319
3395		- if (unlikely(slab_want_init_on_alloc(flags, cachep)) && objp)
3396		- memset(objp, 0, cachep->object_size);
3397		-
3398		- slab_post_alloc_hook(cachep, flags, 1, &objp);
	3320	+out:
	3321	+ slab_post_alloc_hook(cachep, objcg, flags, 1, &objp, init);
3399	3322	return objp;
3400	3323	}
3401	3324
..	..	@@ -3419,29 +3342,29 @@
3419	3342	objp = objpp[i];
3420	3343
3421	3344	page = virt_to_head_page(objp);
3422		- list_del(&page->lru);
	3345	+ list_del(&page->slab_list);
3423	3346	check_spinlock_acquired_node(cachep, node);
3424	3347	slab_put_obj(cachep, page, objp);
3425	3348	STATS_DEC_ACTIVE(cachep);
3426	3349
3427	3350	/* fixup slab chains */
3428	3351	if (page->active == 0) {
3429		- list_add(&page->lru, &n->slabs_free);
	3352	+ list_add(&page->slab_list, &n->slabs_free);
3430	3353	n->free_slabs++;
3431	3354	} else {
3432	3355	/* Unconditionally move a slab to the end of the
3433	3356	* partial list on free - maximum time for the
3434	3357	* other objects to be freed, too.
3435	3358	*/
3436		- list_add_tail(&page->lru, &n->slabs_partial);
	3359	+ list_add_tail(&page->slab_list, &n->slabs_partial);
3437	3360	}
3438	3361	}
3439	3362
3440	3363	while (n->free_objects > n->free_limit && !list_empty(&n->slabs_free)) {
3441	3364	n->free_objects -= cachep->num;
3442	3365
3443		- page = list_last_entry(&n->slabs_free, struct page, lru);
3444		- list_move(&page->lru, list);
	3366	+ page = list_last_entry(&n->slabs_free, struct page, slab_list);
	3367	+ list_move(&page->slab_list, list);
3445	3368	n->free_slabs--;
3446	3369	n->total_slabs--;
3447	3370	}
..	..	@@ -3458,7 +3381,7 @@
3458	3381
3459	3382	check_irq_off();
3460	3383	n = get_node(cachep, node);
3461		- raw_spin_lock(&n->list_lock);
	3384	+ spin_lock(&n->list_lock);
3462	3385	if (n->shared) {
3463	3386	struct array_cache *shared_array = n->shared;
3464	3387	int max = shared_array->limit - shared_array->avail;
..	..	@@ -3479,7 +3402,7 @@
3479	3402	int i = 0;
3480	3403	struct page *page;
3481	3404
3482		- list_for_each_entry(page, &n->slabs_free, lru) {
	3405	+ list_for_each_entry(page, &n->slabs_free, slab_list) {
3483	3406	BUG_ON(page->active);
3484	3407
3485	3408	i++;
..	..	@@ -3487,10 +3410,10 @@
3487	3410	STATS_SET_FREEABLE(cachep, i);
3488	3411	}
3489	3412	#endif
3490		- raw_spin_unlock(&n->list_lock);
3491		- slabs_destroy(cachep, &list);
	3413	+ spin_unlock(&n->list_lock);
3492	3414	ac->avail -= batchcount;
3493	3415	memmove(ac->entry, &(ac->entry[batchcount]), sizeof(void )ac->avail);
	3416	+ slabs_destroy(cachep, &list);
3494	3417	}
3495	3418
3496	3419	/*
..	..	@@ -3500,9 +3423,31 @@
3500	3423	static __always_inline void __cache_free(struct kmem_cache cachep, void objp,
3501	3424	unsigned long caller)
3502	3425	{
3503		- /* Put the object into the quarantine, don't touch it for now. */
3504		- if (kasan_slab_free(cachep, objp, _RET_IP_))
	3426	+ bool init;
	3427	+
	3428	+ if (is_kfence_address(objp)) {
	3429	+ kmemleak_free_recursive(objp, cachep->flags);
	3430	+ memcg_slab_free_hook(cachep, &objp, 1);
	3431	+ __kfence_free(objp);
3505	3432	return;
	3433	+ }
	3434	+
	3435	+ /*
	3436	+ * As memory initialization might be integrated into KASAN,
	3437	+ * kasan_slab_free and initialization memset must be
	3438	+ * kept together to avoid discrepancies in behavior.
	3439	+ */
	3440	+ init = slab_want_init_on_free(cachep);
	3441	+ if (init && !kasan_has_integrated_init())
	3442	+ memset(objp, 0, cachep->object_size);
	3443	+ /* KASAN might put objp into memory quarantine, delaying its reuse. */
	3444	+ if (kasan_slab_free(cachep, objp, init))
	3445	+ return;
	3446	+
	3447	+ /* Use KCSAN to help debug racy use-after-free. */
	3448	+ if (!(cachep->flags & SLAB_TYPESAFE_BY_RCU))
	3449	+ __kcsan_check_access(objp, cachep->object_size,
	3450	+ KCSAN_ACCESS_WRITE \| KCSAN_ACCESS_ASSERT);
3506	3451
3507	3452	___cache_free(cachep, objp, caller);
3508	3453	}
..	..	@@ -3513,10 +3458,9 @@
3513	3458	struct array_cache *ac = cpu_cache_get(cachep);
3514	3459
3515	3460	check_irq_off();
3516		- if (unlikely(slab_want_init_on_free(cachep)))
3517		- memset(objp, 0, cachep->object_size);
3518	3461	kmemleak_free_recursive(objp, cachep->flags);
3519	3462	objp = cache_free_debugcheck(cachep, objp, caller);
	3463	+ memcg_slab_free_hook(cachep, &objp, 1);
3520	3464
3521	3465	/*
3522	3466	* Skip calling cache_free_alien() when the platform is not numa.
..	..	@@ -3544,7 +3488,7 @@
3544	3488	}
3545	3489	}
3546	3490
3547		- ac->entry[ac->avail++] = objp;
	3491	+ __free_one(ac, objp);
3548	3492	}
3549	3493
3550	3494	/**
..	..	@@ -3554,10 +3498,12 @@
3554	3498	*
3555	3499	* Allocate an object from this cache. The flags are only relevant
3556	3500	* if the cache has no available objects.
	3501	+ *
	3502	+ * Return: pointer to the new object or %NULL in case of error
3557	3503	*/
3558	3504	void kmem_cache_alloc(struct kmem_cache cachep, gfp_t flags)
3559	3505	{
3560		- void *ret = slab_alloc(cachep, flags, _RET_IP_);
	3506	+ void *ret = slab_alloc(cachep, flags, cachep->object_size, _RET_IP_);
3561	3507
3562	3508	trace_kmem_cache_alloc(_RET_IP_, ret,
3563	3509	cachep->object_size, cachep->size, flags);
..	..	@@ -3580,8 +3526,9 @@
3580	3526	void **p)
3581	3527	{
3582	3528	size_t i;
	3529	+ struct obj_cgroup *objcg = NULL;
3583	3530
3584		- s = slab_pre_alloc_hook(s, flags);
	3531	+ s = slab_pre_alloc_hook(s, &objcg, size, flags);
3585	3532	if (!s)
3586	3533	return 0;
3587	3534
..	..	@@ -3589,7 +3536,7 @@
3589	3536
3590	3537	local_irq_disable();
3591	3538	for (i = 0; i < size; i++) {
3592		- void *objp = __do_cache_alloc(s, flags);
	3539	+ void *objp = kfence_alloc(s, s->object_size, flags) ?: __do_cache_alloc(s, flags);
3593	3540
3594	3541	if (unlikely(!objp))
3595	3542	goto error;
..	..	@@ -3599,18 +3546,18 @@
3599	3546
3600	3547	cache_alloc_debugcheck_after_bulk(s, flags, size, p, _RET_IP_);
3601	3548
3602		- /* Clear memory outside IRQ disabled section */
3603		- if (unlikely(slab_want_init_on_alloc(flags, s)))
3604		- for (i = 0; i < size; i++)
3605		- memset(p[i], 0, s->object_size);
3606		-
3607		- slab_post_alloc_hook(s, flags, size, p);
	3549	+ /*
	3550	+ * memcg and kmem_cache debug support and memory initialization.
	3551	+ * Done outside of the IRQ disabled section.
	3552	+ */
	3553	+ slab_post_alloc_hook(s, objcg, flags, size, p,
	3554	+ slab_want_init_on_alloc(flags, s));
3608	3555	/* FIXME: Trace call missing. Christoph would like a bulk variant */
3609	3556	return size;
3610	3557	error:
3611	3558	local_irq_enable();
3612	3559	cache_alloc_debugcheck_after_bulk(s, flags, i, p, _RET_IP_);
3613		- slab_post_alloc_hook(s, flags, i, p);
	3560	+ slab_post_alloc_hook(s, objcg, flags, i, p, false);
3614	3561	__kmem_cache_free_bulk(s, i, p);
3615	3562	return 0;
3616	3563	}
..	..	@@ -3622,7 +3569,7 @@
3622	3569	{
3623	3570	void *ret;
3624	3571
3625		- ret = slab_alloc(cachep, flags, _RET_IP_);
	3572	+ ret = slab_alloc(cachep, flags, size, _RET_IP_);
3626	3573
3627	3574	ret = kasan_kmalloc(cachep, ret, size, flags);
3628	3575	trace_kmalloc(_RET_IP_, ret,
..	..	@@ -3643,10 +3590,12 @@
3643	3590	* node, which can improve the performance for cpu bound structures.
3644	3591	*
3645	3592	* Fallback to other node is possible if __GFP_THISNODE is not set.
	3593	+ *
	3594	+ * Return: pointer to the new object or %NULL in case of error
3646	3595	*/
3647	3596	void kmem_cache_alloc_node(struct kmem_cache cachep, gfp_t flags, int nodeid)
3648	3597	{
3649		- void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
	3598	+ void *ret = slab_alloc_node(cachep, flags, nodeid, cachep->object_size, _RET_IP_);
3650	3599
3651	3600	trace_kmem_cache_alloc_node(_RET_IP_, ret,
3652	3601	cachep->object_size, cachep->size,
..	..	@@ -3664,7 +3613,7 @@
3664	3613	{
3665	3614	void *ret;
3666	3615
3667		- ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
	3616	+ ret = slab_alloc_node(cachep, flags, nodeid, size, _RET_IP_);
3668	3617
3669	3618	ret = kasan_kmalloc(cachep, ret, size, flags);
3670	3619	trace_kmalloc_node(_RET_IP_, ret,
..	..	@@ -3711,6 +3660,8 @@
3711	3660	* @size: how many bytes of memory are required.
3712	3661	* @flags: the type of memory to allocate (see kmalloc).
3713	3662	* @caller: function caller for debug tracking of the caller
	3663	+ *
	3664	+ * Return: pointer to the allocated memory or %NULL in case of error
3714	3665	*/
3715	3666	static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
3716	3667	unsigned long caller)
..	..	@@ -3723,7 +3674,7 @@
3723	3674	cachep = kmalloc_slab(size, flags);
3724	3675	if (unlikely(ZERO_OR_NULL_PTR(cachep)))
3725	3676	return cachep;
3726		- ret = slab_alloc(cachep, flags, caller);
	3677	+ ret = slab_alloc(cachep, flags, size, caller);
3727	3678
3728	3679	ret = kasan_kmalloc(cachep, ret, size, flags);
3729	3680	trace_kmalloc(caller, ret,
..	..	@@ -3783,6 +3734,8 @@
3783	3734	s = virt_to_cache(objp);
3784	3735	else
3785	3736	s = cache_from_obj(orig_s, objp);
	3737	+ if (!s)
	3738	+ continue;
3786	3739
3787	3740	debug_check_no_locks_freed(objp, s->object_size);
3788	3741	if (!(s->flags & SLAB_DEBUG_OBJECTS))
..	..	@@ -3817,6 +3770,10 @@
3817	3770	local_irq_save(flags);
3818	3771	kfree_debugcheck(objp);
3819	3772	c = virt_to_cache(objp);
	3773	+ if (!c) {
	3774	+ local_irq_restore(flags);
	3775	+ return;
	3776	+ }
3820	3777	debug_check_no_locks_freed(objp, c->object_size);
3821	3778
3822	3779	debug_check_no_obj_freed(objp, c->object_size);
..	..	@@ -3862,8 +3819,8 @@
3862	3819	}
3863	3820
3864	3821	/* Always called with the slab_mutex held */
3865		-static int __do_tune_cpucache(struct kmem_cache *cachep, int limit,
3866		- int batchcount, int shared, gfp_t gfp)
	3822	+static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
	3823	+ int batchcount, int shared, gfp_t gfp)
3867	3824	{
3868	3825	struct array_cache __percpu cpu_cache, prev;
3869	3826	int cpu;
..	..	@@ -3897,38 +3854,15 @@
3897	3854
3898	3855	node = cpu_to_mem(cpu);
3899	3856	n = get_node(cachep, node);
3900		- raw_spin_lock_irq(&n->list_lock);
	3857	+ spin_lock_irq(&n->list_lock);
3901	3858	free_block(cachep, ac->entry, ac->avail, node, &list);
3902		- raw_spin_unlock_irq(&n->list_lock);
	3859	+ spin_unlock_irq(&n->list_lock);
3903	3860	slabs_destroy(cachep, &list);
3904	3861	}
3905	3862	free_percpu(prev);
3906	3863
3907	3864	setup_node:
3908	3865	return setup_kmem_cache_nodes(cachep, gfp);
3909		-}
3910		-
3911		-static int do_tune_cpucache(struct kmem_cache *cachep, int limit,
3912		- int batchcount, int shared, gfp_t gfp)
3913		-{
3914		- int ret;
3915		- struct kmem_cache *c;
3916		-
3917		- ret = __do_tune_cpucache(cachep, limit, batchcount, shared, gfp);
3918		-
3919		- if (slab_state < FULL)
3920		- return ret;
3921		-
3922		- if ((ret < 0) \|\| !is_root_cache(cachep))
3923		- return ret;
3924		-
3925		- lockdep_assert_held(&slab_mutex);
3926		- for_each_memcg_cache(c, cachep) {
3927		- /* return value determined by the root cache only */
3928		- __do_tune_cpucache(c, limit, batchcount, shared, gfp);
3929		- }
3930		-
3931		- return ret;
3932	3866	}
3933	3867
3934	3868	/* Called with slab_mutex held always */
..	..	@@ -3942,13 +3876,6 @@
3942	3876	err = cache_random_seq_create(cachep, cachep->num, gfp);
3943	3877	if (err)
3944	3878	goto end;
3945		-
3946		- if (!is_root_cache(cachep)) {
3947		- struct kmem_cache *root = memcg_root_cache(cachep);
3948		- limit = root->limit;
3949		- shared = root->shared;
3950		- batchcount = root->batchcount;
3951		- }
3952	3879
3953	3880	if (limit && shared && batchcount)
3954	3881	goto skip_setup;
..	..	@@ -4024,9 +3951,9 @@
4024	3951	return;
4025	3952	}
4026	3953
4027		- raw_spin_lock_irq(&n->list_lock);
	3954	+ spin_lock_irq(&n->list_lock);
4028	3955	drain_array_locked(cachep, ac, node, false, &list);
4029		- raw_spin_unlock_irq(&n->list_lock);
	3956	+ spin_unlock_irq(&n->list_lock);
4030	3957
4031	3958	slabs_destroy(cachep, &list);
4032	3959	}
..	..	@@ -4110,7 +4037,7 @@
4110	4037
4111	4038	for_each_kmem_cache_node(cachep, node, n) {
4112	4039	check_irq_on();
4113		- raw_spin_lock_irq(&n->list_lock);
	4040	+ spin_lock_irq(&n->list_lock);
4114	4041
4115	4042	total_slabs += n->total_slabs;
4116	4043	free_slabs += n->free_slabs;
..	..	@@ -4119,7 +4046,7 @@
4119	4046	if (n->shared)
4120	4047	shared_avail += n->shared->avail;
4121	4048
4122		- raw_spin_unlock_irq(&n->list_lock);
	4049	+ spin_unlock_irq(&n->list_lock);
4123	4050	}
4124	4051	num_objs = total_slabs * cachep->num;
4125	4052	active_slabs = total_slabs - free_slabs;
..	..	@@ -4136,6 +4063,7 @@
4136	4063	sinfo->objects_per_slab = cachep->num;
4137	4064	sinfo->cache_order = cachep->gfporder;
4138	4065	}
	4066	+EXPORT_SYMBOL_GPL(get_slabinfo);
4139	4067
4140	4068	void slabinfo_show_stats(struct seq_file m, struct kmem_cache cachep)
4141	4069	{
..	..	@@ -4176,6 +4104,8 @@
4176	4104	* @buffer: user buffer
4177	4105	* @count: data length
4178	4106	* @ppos: unused
	4107	+ *
	4108	+ * Return: %0 on success, negative error code otherwise.
4179	4109	*/
4180	4110	ssize_t slabinfo_write(struct file file, const char __user buffer,
4181	4111	size_t count, loff_t *ppos)
..	..	@@ -4220,200 +4150,6 @@
4220	4150	return res;
4221	4151	}
4222	4152
4223		-#ifdef CONFIG_DEBUG_SLAB_LEAK
4224		-
4225		-static inline int add_caller(unsigned long *n, unsigned long v)
4226		-{
4227		- unsigned long *p;
4228		- int l;
4229		- if (!v)
4230		- return 1;
4231		- l = n[1];
4232		- p = n + 2;
4233		- while (l) {
4234		- int i = l/2;
4235		- unsigned long q = p + 2 i;
4236		- if (*q == v) {
4237		- q[1]++;
4238		- return 1;
4239		- }
4240		- if (*q > v) {
4241		- l = i;
4242		- } else {
4243		- p = q + 2;
4244		- l -= i + 1;
4245		- }
4246		- }
4247		- if (++n[1] == n[0])
4248		- return 0;
4249		- memmove(p + 2, p, n[1] * 2 * sizeof(unsigned long) - ((void )p - (void )n));
4250		- p[0] = v;
4251		- p[1] = 1;
4252		- return 1;
4253		-}
4254		-
4255		-static void handle_slab(unsigned long n, struct kmem_cache c,
4256		- struct page *page)
4257		-{
4258		- void *p;
4259		- int i, j;
4260		- unsigned long v;
4261		-
4262		- if (n[0] == n[1])
4263		- return;
4264		- for (i = 0, p = page->s_mem; i < c->num; i++, p += c->size) {
4265		- bool active = true;
4266		-
4267		- for (j = page->active; j < c->num; j++) {
4268		- if (get_free_obj(page, j) == i) {
4269		- active = false;
4270		- break;
4271		- }
4272		- }
4273		-
4274		- if (!active)
4275		- continue;
4276		-
4277		- /*
4278		- * probe_kernel_read() is used for DEBUG_PAGEALLOC. page table
4279		- * mapping is established when actual object allocation and
4280		- * we could mistakenly access the unmapped object in the cpu
4281		- * cache.
4282		- */
4283		- if (probe_kernel_read(&v, dbg_userword(c, p), sizeof(v)))
4284		- continue;
4285		-
4286		- if (!add_caller(n, v))
4287		- return;
4288		- }
4289		-}
4290		-
4291		-static void show_symbol(struct seq_file *m, unsigned long address)
4292		-{
4293		-#ifdef CONFIG_KALLSYMS
4294		- unsigned long offset, size;
4295		- char modname[MODULE_NAME_LEN], name[KSYM_NAME_LEN];
4296		-
4297		- if (lookup_symbol_attrs(address, &size, &offset, modname, name) == 0) {
4298		- seq_printf(m, "%s+%#lx/%#lx", name, offset, size);
4299		- if (modname[0])
4300		- seq_printf(m, " [%s]", modname);
4301		- return;
4302		- }
4303		-#endif
4304		- seq_printf(m, "%px", (void *)address);
4305		-}
4306		-
4307		-static int leaks_show(struct seq_file m, void p)
4308		-{
4309		- struct kmem_cache *cachep = list_entry(p, struct kmem_cache,
4310		- root_caches_node);
4311		- struct page *page;
4312		- struct kmem_cache_node *n;
4313		- const char *name;
4314		- unsigned long *x = m->private;
4315		- int node;
4316		- int i;
4317		-
4318		- if (!(cachep->flags & SLAB_STORE_USER))
4319		- return 0;
4320		- if (!(cachep->flags & SLAB_RED_ZONE))
4321		- return 0;
4322		-
4323		- /*
4324		- * Set store_user_clean and start to grab stored user information
4325		- * for all objects on this cache. If some alloc/free requests comes
4326		- * during the processing, information would be wrong so restart
4327		- * whole processing.
4328		- */
4329		- do {
4330		- drain_cpu_caches(cachep);
4331		- /*
4332		- * drain_cpu_caches() could make kmemleak_object and
4333		- * debug_objects_cache dirty, so reset afterwards.
4334		- */
4335		- set_store_user_clean(cachep);
4336		-
4337		- x[1] = 0;
4338		-
4339		- for_each_kmem_cache_node(cachep, node, n) {
4340		-
4341		- check_irq_on();
4342		- raw_spin_lock_irq(&n->list_lock);
4343		-
4344		- list_for_each_entry(page, &n->slabs_full, lru)
4345		- handle_slab(x, cachep, page);
4346		- list_for_each_entry(page, &n->slabs_partial, lru)
4347		- handle_slab(x, cachep, page);
4348		- raw_spin_unlock_irq(&n->list_lock);
4349		- }
4350		- } while (!is_store_user_clean(cachep));
4351		-
4352		- name = cachep->name;
4353		- if (x[0] == x[1]) {
4354		- /* Increase the buffer size */
4355		- mutex_unlock(&slab_mutex);
4356		- m->private = kcalloc(x[0] * 4, sizeof(unsigned long),
4357		- GFP_KERNEL);
4358		- if (!m->private) {
4359		- /* Too bad, we are really out */
4360		- m->private = x;
4361		- mutex_lock(&slab_mutex);
4362		- return -ENOMEM;
4363		- }
4364		- (unsigned long )m->private = x[0] * 2;
4365		- kfree(x);
4366		- mutex_lock(&slab_mutex);
4367		- /* Now make sure this entry will be retried */
4368		- m->count = m->size;
4369		- return 0;
4370		- }
4371		- for (i = 0; i < x[1]; i++) {
4372		- seq_printf(m, "%s: %lu ", name, x[2*i+3]);
4373		- show_symbol(m, x[2*i+2]);
4374		- seq_putc(m, '\n');
4375		- }
4376		-
4377		- return 0;
4378		-}
4379		-
4380		-static const struct seq_operations slabstats_op = {
4381		- .start = slab_start,
4382		- .next = slab_next,
4383		- .stop = slab_stop,
4384		- .show = leaks_show,
4385		-};
4386		-
4387		-static int slabstats_open(struct inode inode, struct file file)
4388		-{
4389		- unsigned long *n;
4390		-
4391		- n = __seq_open_private(file, &slabstats_op, PAGE_SIZE);
4392		- if (!n)
4393		- return -ENOMEM;
4394		-
4395		- n = PAGE_SIZE / (2 sizeof(unsigned long));
4396		-
4397		- return 0;
4398		-}
4399		-
4400		-static const struct file_operations proc_slabstats_operations = {
4401		- .open = slabstats_open,
4402		- .read = seq_read,
4403		- .llseek = seq_lseek,
4404		- .release = seq_release_private,
4405		-};
4406		-#endif
4407		-
4408		-static int __init slab_proc_init(void)
4409		-{
4410		-#ifdef CONFIG_DEBUG_SLAB_LEAK
4411		- proc_create("slab_allocators", 0, NULL, &proc_slabstats_operations);
4412		-#endif
4413		- return 0;
4414		-}
4415		-module_init(slab_proc_init);
4416		-
4417	4153	#ifdef CONFIG_HARDENED_USERCOPY
4418	4154	/*
4419	4155	* Rejects incorrectly sized objects and objects that are to be copied
..	..	@@ -4438,7 +4174,10 @@
4438	4174	BUG_ON(objnr >= cachep->num);
4439	4175
4440	4176	/* Find offset within object. */
4441		- offset = ptr - index_to_obj(cachep, page, objnr) - obj_offset(cachep);
	4177	+ if (is_kfence_address(ptr))
	4178	+ offset = ptr - kfence_object_start(ptr);
	4179	+ else
	4180	+ offset = ptr - index_to_obj(cachep, page, objnr) - obj_offset(cachep);
4442	4181
4443	4182	/* Allow address range falling entirely within usercopy region. */
4444	4183	if (offset >= cachep->useroffset &&
..	..	@@ -4464,31 +4203,26 @@
4464	4203	#endif /* CONFIG_HARDENED_USERCOPY */
4465	4204
4466	4205	/**
4467		- * ksize - get the actual amount of memory allocated for a given object
4468		- * @objp: Pointer to the object
	4206	+ * __ksize -- Uninstrumented ksize.
	4207	+ * @objp: pointer to the object
4469	4208	*
4470		- * kmalloc may internally round up allocations and return more memory
4471		- * than requested. ksize() can be used to determine the actual amount of
4472		- * memory allocated. The caller may use this additional memory, even though
4473		- * a smaller amount of memory was initially specified with the kmalloc call.
4474		- * The caller must guarantee that objp points to a valid object previously
4475		- * allocated with either kmalloc() or kmem_cache_alloc(). The object
4476		- * must not be freed during the duration of the call.
	4209	+ * Unlike ksize(), __ksize() is uninstrumented, and does not provide the same
	4210	+ * safety checks as ksize() with KASAN instrumentation enabled.
	4211	+ *
	4212	+ * Return: size of the actual memory used by @objp in bytes
4477	4213	*/
4478		-size_t ksize(const void *objp)
	4214	+size_t __ksize(const void *objp)
4479	4215	{
	4216	+ struct kmem_cache *c;
4480	4217	size_t size;
4481	4218
4482	4219	BUG_ON(!objp);
4483	4220	if (unlikely(objp == ZERO_SIZE_PTR))
4484	4221	return 0;
4485	4222
4486		- size = virt_to_cache(objp)->object_size;
4487		- /* We assume that ksize callers could use the whole allocated area,
4488		- * so we need to unpoison this area.
4489		- */
4490		- kasan_unpoison_shadow(objp, size);
	4223	+ c = virt_to_cache(objp);
	4224	+ size = c ? c->object_size : 0;
4491	4225
4492	4226	return size;
4493	4227	}
4494		-EXPORT_SYMBOL(ksize);
	4228	+EXPORT_SYMBOL(__ksize);