forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-02-20 102a0743326a03cd1a1202ceda21e175b7d3575c 
 kernel/fs/dcache.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0-only
12 /*
23  * fs/dcache.c
34  *
....@@ -27,7 +28,7 @@
2728 #include <linux/export.h>
2829 #include <linux/security.h>
2930 #include <linux/seqlock.h>
30
-#include <linux/bootmem.h>
31
+#include <linux/memblock.h>
3132 #include <linux/bit_spinlock.h>
3233 #include <linux/rculist_bl.h>
3334 #include <linux/list_lru.h>
....@@ -120,6 +121,7 @@
120121 
121122 static DEFINE_PER_CPU(long, nr_dentry);
122123 static DEFINE_PER_CPU(long, nr_dentry_unused);
124
+static DEFINE_PER_CPU(long, nr_dentry_negative);
123125 
124126 #if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
125127 
....@@ -153,11 +155,22 @@
153155 	return sum < 0 ? 0 : sum;
154156 }
155157 
156
-int proc_nr_dentry(struct ctl_table *table, int write, void __user *buffer,
158
+static long get_nr_dentry_negative(void)
159
+{
160
+	int i;
161
+	long sum = 0;
162
+
163
+	for_each_possible_cpu(i)
164
+		sum += per_cpu(nr_dentry_negative, i);
165
+	return sum < 0 ? 0 : sum;
166
+}
167
+
168
+int proc_nr_dentry(struct ctl_table *table, int write, void *buffer,
157169 		   size_t *lenp, loff_t *ppos)
158170 {
159171 	dentry_stat.nr_dentry = get_nr_dentry();
160172 	dentry_stat.nr_unused = get_nr_dentry_unused();
173
+	dentry_stat.nr_negative = get_nr_dentry_negative();
161174 	return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
162175 }
163176 #endif
....@@ -273,25 +286,23 @@
273286 void take_dentry_name_snapshot(struct name_snapshot *name, struct dentry *dentry)
274287 {
275288 	spin_lock(&dentry->d_lock);
289
+	name->name = dentry->d_name;
276290 	if (unlikely(dname_external(dentry))) {
277
-		struct external_name *p = external_name(dentry);
278
-		atomic_inc(&p->u.count);
279
-		spin_unlock(&dentry->d_lock);
280
-		name->name = p->name;
291
+		atomic_inc(&external_name(dentry)->u.count);
281292 	} else {
282293 		memcpy(name->inline_name, dentry->d_iname,
283294 		       dentry->d_name.len + 1);
284
-		spin_unlock(&dentry->d_lock);
285
-		name->name = name->inline_name;
295
+		name->name.name = name->inline_name;
286296 	}
297
+	spin_unlock(&dentry->d_lock);
287298 }
288299 EXPORT_SYMBOL(take_dentry_name_snapshot);
289300 
290301 void release_dentry_name_snapshot(struct name_snapshot *name)
291302 {
292
-	if (unlikely(name->name != name->inline_name)) {
303
+	if (unlikely(name->name.name != name->inline_name)) {
293304 		struct external_name *p;
294
-		p = container_of(name->name, struct external_name, name[0]);
305
+		p = container_of(name->name.name, struct external_name, name[0]);
295306 		if (unlikely(atomic_dec_and_test(&p->u.count)))
296307 			kfree_rcu(p, u.head);
297308 	}
....@@ -308,7 +319,7 @@
308319 	flags = READ_ONCE(dentry->d_flags);
309320 	flags &= ~(DCACHE_ENTRY_TYPE | DCACHE_FALLTHRU);
310321 	flags |= type_flags;
311
-	WRITE_ONCE(dentry->d_flags, flags);
322
+	smp_store_release(&dentry->d_flags, flags);
312323 }
313324 
314325 static inline void __d_clear_type_and_inode(struct dentry *dentry)
....@@ -318,6 +329,8 @@
318329 	flags &= ~(DCACHE_ENTRY_TYPE | DCACHE_FALLTHRU);
319330 	WRITE_ONCE(dentry->d_flags, flags);
320331 	dentry->d_inode = NULL;
332
+	if (dentry->d_flags & DCACHE_LRU_LIST)
333
+		this_cpu_inc(nr_dentry_negative);
321334 }
322335 
323336 static void dentry_free(struct dentry *dentry)
....@@ -372,6 +385,11 @@
372385  * The per-cpu "nr_dentry_unused" counters are updated with
373386  * the DCACHE_LRU_LIST bit.
374387  *
388
+ * The per-cpu "nr_dentry_negative" counters are only updated
389
+ * when deleted from or added to the per-superblock LRU list, not
390
+ * from/to the shrink list. That is to avoid an unneeded dec/inc
391
+ * pair when moving from LRU to shrink list in select_collect().
392
+ *
375393  * These helper functions make sure we always follow the
376394  * rules. d_lock must be held by the caller.
377395  */
....@@ -381,6 +399,8 @@
381399 	D_FLAG_VERIFY(dentry, 0);
382400 	dentry->d_flags |= DCACHE_LRU_LIST;
383401 	this_cpu_inc(nr_dentry_unused);
402
+	if (d_is_negative(dentry))
403
+		this_cpu_inc(nr_dentry_negative);
384404 	WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
385405 }
386406 
....@@ -389,6 +409,8 @@
389409 	D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
390410 	dentry->d_flags &= ~DCACHE_LRU_LIST;
391411 	this_cpu_dec(nr_dentry_unused);
412
+	if (d_is_negative(dentry))
413
+		this_cpu_dec(nr_dentry_negative);
392414 	WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
393415 }
394416 
....@@ -419,6 +441,8 @@
419441 	D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
420442 	dentry->d_flags &= ~DCACHE_LRU_LIST;
421443 	this_cpu_dec(nr_dentry_unused);
444
+	if (d_is_negative(dentry))
445
+		this_cpu_dec(nr_dentry_negative);
422446 	list_lru_isolate(lru, &dentry->d_lru);
423447 }
424448 
....@@ -427,6 +451,8 @@
427451 {
428452 	D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
429453 	dentry->d_flags |= DCACHE_SHRINK_LIST;
454
+	if (d_is_negative(dentry))
455
+		this_cpu_dec(nr_dentry_negative);
430456 	list_lru_isolate_move(lru, &dentry->d_lru, list);
431457 }
432458 
....@@ -621,6 +647,10 @@
621647 		if (dentry->d_op->d_delete(dentry))
622648 			return false;
623649 	}
650
+
651
+	if (unlikely(dentry->d_flags & DCACHE_DONTCACHE))
652
+		return false;
653
+
624654 	/* retain; LRU fodder */
625655 	dentry->d_lockref.count--;
626656 	if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST)))
....@@ -629,6 +659,21 @@
629659 		dentry->d_flags |= DCACHE_REFERENCED;
630660 	return true;
631661 }
662
+
663
+void d_mark_dontcache(struct inode *inode)
664
+{
665
+	struct dentry *de;
666
+
667
+	spin_lock(&inode->i_lock);
668
+	hlist_for_each_entry(de, &inode->i_dentry, d_u.d_alias) {
669
+		spin_lock(&de->d_lock);
670
+		de->d_flags |= DCACHE_DONTCACHE;
671
+		spin_unlock(&de->d_lock);
672
+	}
673
+	inode->i_state |= I_DONTCACHE;
674
+	spin_unlock(&inode->i_lock);
675
+}
676
+EXPORT_SYMBOL(d_mark_dontcache);
632677 
633678 /*
634679  * Finish off a dentry we've decided to kill.
....@@ -835,6 +880,32 @@
835880 }
836881 EXPORT_SYMBOL(dput);
837882 
883
+static void __dput_to_list(struct dentry *dentry, struct list_head *list)
884
+__must_hold(&dentry->d_lock)
885
+{
886
+	if (dentry->d_flags & DCACHE_SHRINK_LIST) {
887
+		/* let the owner of the list it's on deal with it */
888
+		--dentry->d_lockref.count;
889
+	} else {
890
+		if (dentry->d_flags & DCACHE_LRU_LIST)
891
+			d_lru_del(dentry);
892
+		if (!--dentry->d_lockref.count)
893
+			d_shrink_add(dentry, list);
894
+	}
895
+}
896
+
897
+void dput_to_list(struct dentry *dentry, struct list_head *list)
898
+{
899
+	rcu_read_lock();
900
+	if (likely(fast_dput(dentry))) {
901
+		rcu_read_unlock();
902
+		return;
903
+	}
904
+	rcu_read_unlock();
905
+	if (!retain_dentry(dentry))
906
+		__dput_to_list(dentry, list);
907
+	spin_unlock(&dentry->d_lock);
908
+}
838909 
839910 /* This must be called with d_lock held */
840911 static inline void __dget_dlock(struct dentry *dentry)
....@@ -1043,7 +1114,7 @@
10431114 	return false;
10441115 }
10451116 
1046
-static void shrink_dentry_list(struct list_head *list)
1117
+void shrink_dentry_list(struct list_head *list)
10471118 {
10481119 	while (!list_empty(list)) {
10491120 		struct dentry *dentry, *parent;
....@@ -1065,18 +1136,9 @@
10651136 		rcu_read_unlock();
10661137 		d_shrink_del(dentry);
10671138 		parent = dentry->d_parent;
1139
+		if (parent != dentry)
1140
+			__dput_to_list(parent, list);
10681141 		__dentry_kill(dentry);
1069
-		if (parent == dentry)
1070
-			continue;
1071
-		/*
1072
-		 * We need to prune ancestors too. This is necessary to prevent
1073
-		 * quadratic behavior of shrink_dcache_parent(), but is also
1074
-		 * expected to be beneficial in reducing dentry cache
1075
-		 * fragmentation.
1076
-		 */
1077
-		dentry = parent;
1078
-		while (dentry && !lockref_put_or_lock(&dentry->d_lockref))
1079
-			dentry = dentry_kill(dentry);
10801142 	}
10811143 }
10821144 
....@@ -1278,7 +1340,7 @@
12781340 
12791341 		if (!list_empty(&dentry->d_subdirs)) {
12801342 			spin_unlock(&this_parent->d_lock);
1281
-			spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_);
1343
+			spin_release(&dentry->d_lock.dep_map, _RET_IP_);
12821344 			this_parent = dentry;
12831345 			spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
12841346 			goto repeat;
....@@ -1421,8 +1483,11 @@
14211483 
14221484 struct select_data {
14231485 	struct dentry *start;
1486
+	union {
1487
+		long found;
1488
+		struct dentry *victim;
1489
+	};
14241490 	struct list_head dispose;
1425
-	int found;
14261491 };
14271492 
14281493 static enum d_walk_ret select_collect(void *_data, struct dentry *dentry)
....@@ -1454,6 +1519,37 @@
14541519 	return ret;
14551520 }
14561521 
1522
+static enum d_walk_ret select_collect2(void *_data, struct dentry *dentry)
1523
+{
1524
+	struct select_data *data = _data;
1525
+	enum d_walk_ret ret = D_WALK_CONTINUE;
1526
+
1527
+	if (data->start == dentry)
1528
+		goto out;
1529
+
1530
+	if (dentry->d_flags & DCACHE_SHRINK_LIST) {
1531
+		if (!dentry->d_lockref.count) {
1532
+			rcu_read_lock();
1533
+			data->victim = dentry;
1534
+			return D_WALK_QUIT;
1535
+		}
1536
+	} else {
1537
+		if (dentry->d_flags & DCACHE_LRU_LIST)
1538
+			d_lru_del(dentry);
1539
+		if (!dentry->d_lockref.count)
1540
+			d_shrink_add(dentry, &data->dispose);
1541
+	}
1542
+	/*
1543
+	 * We can return to the caller if we have found some (this
1544
+	 * ensures forward progress). We'll be coming back to find
1545
+	 * the rest.
1546
+	 */
1547
+	if (!list_empty(&data->dispose))
1548
+		ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
1549
+out:
1550
+	return ret;
1551
+}
1552
+
14571553 /**
14581554  * shrink_dcache_parent - prune dcache
14591555  * @parent: parent of entries to prune
....@@ -1463,12 +1559,9 @@
14631559 void shrink_dcache_parent(struct dentry *parent)
14641560 {
14651561 	for (;;) {
1466
-		struct select_data data;
1562
+		struct select_data data = {.start = parent};
14671563 
14681564 		INIT_LIST_HEAD(&data.dispose);
1469
-		data.start = parent;
1470
-		data.found = 0;
1471
-
14721565 		d_walk(parent, &data, select_collect);
14731566 
14741567 		if (!list_empty(&data.dispose)) {
....@@ -1479,6 +1572,24 @@
14791572 		cond_resched();
14801573 		if (!data.found)
14811574 			break;
1575
+		data.victim = NULL;
1576
+		d_walk(parent, &data, select_collect2);
1577
+		if (data.victim) {
1578
+			struct dentry *parent;
1579
+			spin_lock(&data.victim->d_lock);
1580
+			if (!shrink_lock_dentry(data.victim)) {
1581
+				spin_unlock(&data.victim->d_lock);
1582
+				rcu_read_unlock();
1583
+			} else {
1584
+				rcu_read_unlock();
1585
+				parent = data.victim->d_parent;
1586
+				if (parent != data.victim)
1587
+					__dput_to_list(parent, &data.dispose);
1588
+				__dentry_kill(data.victim);
1589
+			}
1590
+		}
1591
+		if (!list_empty(&data.dispose))
1592
+			shrink_dentry_list(&data.dispose);
14821593 	}
14831594 }
14841595 EXPORT_SYMBOL(shrink_dcache_parent);
....@@ -1589,7 +1700,7 @@
15891700  * copied and the copy passed in may be reused after this call.
15901701  */
15911702  
1592
-struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
1703
+static struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
15931704 {
15941705 	struct dentry *dentry;
15951706 	char *dname;
....@@ -1635,7 +1746,7 @@
16351746 	dentry->d_lockref.count = 1;
16361747 	dentry->d_flags = 0;
16371748 	spin_lock_init(&dentry->d_lock);
1638
-	seqcount_init(&dentry->d_seq);
1749
+	seqcount_spinlock_init(&dentry->d_seq, &dentry->d_lock);
16391750 	dentry->d_inode = NULL;
16401751 	dentry->d_parent = dentry;
16411752 	dentry->d_sb = sb;
....@@ -1718,6 +1829,9 @@
17181829  * never be anyone's children or parents.  Unlike all other
17191830  * dentries, these will not have RCU delay between dropping the
17201831  * last reference and freeing them.
1832
+ *
1833
+ * The only user is alloc_file_pseudo() and that's what should
1834
+ * be considered a public interface.  Don't use directly.
17211835  */
17221836 struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name)
17231837 {
....@@ -1726,7 +1840,6 @@
17261840 		dentry->d_flags |= DCACHE_NORCU;
17271841 	return dentry;
17281842 }
1729
-EXPORT_SYMBOL(d_alloc_pseudo);
17301843 
17311844 struct dentry *d_alloc_name(struct dentry *parent, const char *name)
17321845 {
....@@ -1825,6 +1938,11 @@
18251938 	WARN_ON(d_in_lookup(dentry));
18261939 
18271940 	spin_lock(&dentry->d_lock);
1941
+	/*
1942
+	 * Decrement negative dentry count if it was in the LRU list.
1943
+	 */
1944
+	if (dentry->d_flags & DCACHE_LRU_LIST)
1945
+		this_cpu_dec(nr_dentry_negative);
18281946 	hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
18291947 	raw_write_seqcount_begin(&dentry->d_seq);
18301948 	__d_set_inode_and_type(dentry, inode, add_flags);
....@@ -1992,7 +2110,7 @@
19922110 {
19932111 	return __d_obtain_alias(inode, true);
19942112 }
1995
-EXPORT_SYMBOL(d_obtain_alias);
2113
+EXPORT_SYMBOL_NS(d_obtain_alias, ANDROID_GKI_VFS_EXPORT_ONLY);
19962114 
19972115 /**
19982116  * d_obtain_root - find or allocate a dentry for a given inode
....@@ -2066,7 +2184,7 @@
20662184 	}
20672185 	return found;
20682186 }
2069
-EXPORT_SYMBOL(d_add_ci);
2187
+EXPORT_SYMBOL_NS(d_add_ci, ANDROID_GKI_VFS_EXPORT_ONLY);
20702188 
20712189 
20722190 static inline bool d_same_name(const struct dentry *dentry,
....@@ -2341,7 +2459,6 @@
23412459 void d_delete(struct dentry * dentry)
23422460 {
23432461 	struct inode *inode = dentry->d_inode;
2344
-	int isdir = d_is_dir(dentry);
23452462 
23462463 	spin_lock(&inode->i_lock);
23472464 	spin_lock(&dentry->d_lock);
....@@ -2356,7 +2473,6 @@
23562473 		spin_unlock(&dentry->d_lock);
23572474 		spin_unlock(&inode->i_lock);
23582475 	}
2359
-	fsnotify_nameremove(dentry, isdir);
23602476 }
23612477 EXPORT_SYMBOL(d_delete);
23622478 
....@@ -2387,10 +2503,9 @@
23872503 static inline unsigned start_dir_add(struct inode *dir)
23882504 {
23892505 
2390
-	preempt_disable_rt();
23912506 	for (;;) {
2392
-		unsigned n = dir->__i_dir_seq;
2393
-		if (!(n & 1) && cmpxchg(&dir->__i_dir_seq, n, n + 1) == n)
2507
+		unsigned n = dir->i_dir_seq;
2508
+		if (!(n & 1) && cmpxchg(&dir->i_dir_seq, n, n + 1) == n)
23942509 			return n;
23952510 		cpu_relax();
23962511 	}
....@@ -2398,30 +2513,26 @@
23982513 
23992514 static inline void end_dir_add(struct inode *dir, unsigned n)
24002515 {
2401
-	smp_store_release(&dir->__i_dir_seq, n + 2);
2402
-	preempt_enable_rt();
2516
+	smp_store_release(&dir->i_dir_seq, n + 2);
24032517 }
24042518 
24052519 static void d_wait_lookup(struct dentry *dentry)
24062520 {
2407
-	struct swait_queue __wait;
2408
-
2409
-	if (!d_in_lookup(dentry))
2410
-		return;
2411
-
2412
-	INIT_LIST_HEAD(&__wait.task_list);
2413
-	do {
2414
-		prepare_to_swait_exclusive(dentry->d_wait, &__wait, TASK_UNINTERRUPTIBLE);
2415
-		spin_unlock(&dentry->d_lock);
2416
-		schedule();
2417
-		spin_lock(&dentry->d_lock);
2418
-	} while (d_in_lookup(dentry));
2419
-	finish_swait(dentry->d_wait, &__wait);
2521
+	if (d_in_lookup(dentry)) {
2522
+		DECLARE_WAITQUEUE(wait, current);
2523
+		add_wait_queue(dentry->d_wait, &wait);
2524
+		do {
2525
+			set_current_state(TASK_UNINTERRUPTIBLE);
2526
+			spin_unlock(&dentry->d_lock);
2527
+			schedule();
2528
+			spin_lock(&dentry->d_lock);
2529
+		} while (d_in_lookup(dentry));
2530
+	}
24202531 }
24212532 
24222533 struct dentry *d_alloc_parallel(struct dentry *parent,
24232534 				const struct qstr *name,
2424
-				struct swait_queue_head *wq)
2535
+				wait_queue_head_t *wq)
24252536 {
24262537 	unsigned int hash = name->hash;
24272538 	struct hlist_bl_head *b = in_lookup_hash(parent, hash);
....@@ -2435,7 +2546,7 @@
24352546 
24362547 retry:
24372548 	rcu_read_lock();
2438
-	seq = smp_load_acquire(&parent->d_inode->__i_dir_seq);
2549
+	seq = smp_load_acquire(&parent->d_inode->i_dir_seq);
24392550 	r_seq = read_seqbegin(&rename_lock);
24402551 	dentry = __d_lookup_rcu(parent, name, &d_seq);
24412552 	if (unlikely(dentry)) {
....@@ -2463,7 +2574,7 @@
24632574 	}
24642575 
24652576 	hlist_bl_lock(b);
2466
-	if (unlikely(READ_ONCE(parent->d_inode->__i_dir_seq) != seq)) {
2577
+	if (unlikely(READ_ONCE(parent->d_inode->i_dir_seq) != seq)) {
24672578 		hlist_bl_unlock(b);
24682579 		rcu_read_unlock();
24692580 		goto retry;
....@@ -2536,7 +2647,7 @@
25362647 	hlist_bl_lock(b);
25372648 	dentry->d_flags &= ~DCACHE_PAR_LOOKUP;
25382649 	__hlist_bl_del(&dentry->d_u.d_in_lookup_hash);
2539
-	swake_up_all(dentry->d_wait);
2650
+	wake_up_all(dentry->d_wait);
25402651 	dentry->d_wait = NULL;
25412652 	hlist_bl_unlock(b);
25422653 	INIT_HLIST_NODE(&dentry->d_u.d_alias);
....@@ -2954,7 +3065,7 @@
29543065 	__d_add(dentry, inode);
29553066 	return NULL;
29563067 }
2957
-EXPORT_SYMBOL(d_splice_alias);
3068
+EXPORT_SYMBOL_NS(d_splice_alias, ANDROID_GKI_VFS_EXPORT_ONLY);
29583069 
29593070 /*
29603071  * Test whether new_dentry is a subdirectory of old_dentry.
....@@ -3049,8 +3160,6 @@
30493160 
30503161 static void __init dcache_init_early(void)
30513162 {
3052
-	unsigned int loop;
3053
-
30543163 	/* If hashes are distributed across NUMA nodes, defer
30553164 	 * hash allocation until vmalloc space is available.
30563165 	 */
....@@ -3067,16 +3176,11 @@
30673176 					NULL,
30683177 					0,
30693178 					0);
3070
-
3071
-	for (loop = 0; loop < (1U << d_hash_shift); loop++)
3072
-		INIT_HLIST_BL_HEAD(dentry_hashtable + loop);
3073
-
30743179 	d_hash_shift = 32 - d_hash_shift;
30753180 }
30763181 
30773182 static void __init dcache_init(void)
30783183 {
3079
-	unsigned int loop;
30803184 	/*
30813185 	 * A constructor could be added for stable state like the lists,
30823186 	 * but it is probably not worth it because of the cache nature
....@@ -3100,10 +3204,6 @@
31003204 					NULL,
31013205 					0,
31023206 					0);
3103
-
3104
-	for (loop = 0; loop < (1U << d_hash_shift); loop++)
3105
-		INIT_HLIST_BL_HEAD(dentry_hashtable + loop);
3106
-
31073207 	d_hash_shift = 32 - d_hash_shift;
31083208 }
31093209