From 102a0743326a03cd1a1202ceda21e175b7d3575c Mon Sep 17 00:00:00 2001
From: hc <hc@nodka.com>
Date: Tue, 20 Feb 2024 01:20:52 +0000
Subject: [PATCH] add new system file
---
kernel/fs/dcache.c | 190 ++++++++++++++++++++++++++++++++++++++---------
1 files changed, 153 insertions(+), 37 deletions(-)
diff --git a/kernel/fs/dcache.c b/kernel/fs/dcache.c
index d3553ca..cb588ce 100644
--- a/kernel/fs/dcache.c
+++ b/kernel/fs/dcache.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0-only
/*
* fs/dcache.c
*
@@ -27,7 +28,7 @@
#include <linux/export.h>
#include <linux/security.h>
#include <linux/seqlock.h>
-#include <linux/bootmem.h>
+#include <linux/memblock.h>
#include <linux/bit_spinlock.h>
#include <linux/rculist_bl.h>
#include <linux/list_lru.h>
@@ -120,6 +121,7 @@
static DEFINE_PER_CPU(long, nr_dentry);
static DEFINE_PER_CPU(long, nr_dentry_unused);
+static DEFINE_PER_CPU(long, nr_dentry_negative);
#if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
@@ -153,11 +155,22 @@
return sum < 0 ? 0 : sum;
}
-int proc_nr_dentry(struct ctl_table *table, int write, void __user *buffer,
+static long get_nr_dentry_negative(void)
+{
+ int i;
+ long sum = 0;
+
+ for_each_possible_cpu(i)
+ sum += per_cpu(nr_dentry_negative, i);
+ return sum < 0 ? 0 : sum;
+}
+
+int proc_nr_dentry(struct ctl_table *table, int write, void *buffer,
size_t *lenp, loff_t *ppos)
{
dentry_stat.nr_dentry = get_nr_dentry();
dentry_stat.nr_unused = get_nr_dentry_unused();
+ dentry_stat.nr_negative = get_nr_dentry_negative();
return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
}
#endif
@@ -273,25 +286,23 @@
void take_dentry_name_snapshot(struct name_snapshot *name, struct dentry *dentry)
{
spin_lock(&dentry->d_lock);
+ name->name = dentry->d_name;
if (unlikely(dname_external(dentry))) {
- struct external_name *p = external_name(dentry);
- atomic_inc(&p->u.count);
- spin_unlock(&dentry->d_lock);
- name->name = p->name;
+ atomic_inc(&external_name(dentry)->u.count);
} else {
memcpy(name->inline_name, dentry->d_iname,
dentry->d_name.len + 1);
- spin_unlock(&dentry->d_lock);
- name->name = name->inline_name;
+ name->name.name = name->inline_name;
}
+ spin_unlock(&dentry->d_lock);
}
EXPORT_SYMBOL(take_dentry_name_snapshot);
void release_dentry_name_snapshot(struct name_snapshot *name)
{
- if (unlikely(name->name != name->inline_name)) {
+ if (unlikely(name->name.name != name->inline_name)) {
struct external_name *p;
- p = container_of(name->name, struct external_name, name[0]);
+ p = container_of(name->name.name, struct external_name, name[0]);
if (unlikely(atomic_dec_and_test(&p->u.count)))
kfree_rcu(p, u.head);
}
@@ -308,7 +319,7 @@
flags = READ_ONCE(dentry->d_flags);
flags &= ~(DCACHE_ENTRY_TYPE | DCACHE_FALLTHRU);
flags |= type_flags;
- WRITE_ONCE(dentry->d_flags, flags);
+ smp_store_release(&dentry->d_flags, flags);
}
static inline void __d_clear_type_and_inode(struct dentry *dentry)
@@ -318,6 +329,8 @@
flags &= ~(DCACHE_ENTRY_TYPE | DCACHE_FALLTHRU);
WRITE_ONCE(dentry->d_flags, flags);
dentry->d_inode = NULL;
+ if (dentry->d_flags & DCACHE_LRU_LIST)
+ this_cpu_inc(nr_dentry_negative);
}
static void dentry_free(struct dentry *dentry)
@@ -372,6 +385,11 @@
* The per-cpu "nr_dentry_unused" counters are updated with
* the DCACHE_LRU_LIST bit.
*
+ * The per-cpu "nr_dentry_negative" counters are only updated
+ * when deleted from or added to the per-superblock LRU list, not
+ * from/to the shrink list. That is to avoid an unneeded dec/inc
+ * pair when moving from LRU to shrink list in select_collect().
+ *
* These helper functions make sure we always follow the
* rules. d_lock must be held by the caller.
*/
@@ -381,6 +399,8 @@
D_FLAG_VERIFY(dentry, 0);
dentry->d_flags |= DCACHE_LRU_LIST;
this_cpu_inc(nr_dentry_unused);
+ if (d_is_negative(dentry))
+ this_cpu_inc(nr_dentry_negative);
WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
}
@@ -389,6 +409,8 @@
D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
dentry->d_flags &= ~DCACHE_LRU_LIST;
this_cpu_dec(nr_dentry_unused);
+ if (d_is_negative(dentry))
+ this_cpu_dec(nr_dentry_negative);
WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
}
@@ -419,6 +441,8 @@
D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
dentry->d_flags &= ~DCACHE_LRU_LIST;
this_cpu_dec(nr_dentry_unused);
+ if (d_is_negative(dentry))
+ this_cpu_dec(nr_dentry_negative);
list_lru_isolate(lru, &dentry->d_lru);
}
@@ -427,6 +451,8 @@
{
D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
dentry->d_flags |= DCACHE_SHRINK_LIST;
+ if (d_is_negative(dentry))
+ this_cpu_dec(nr_dentry_negative);
list_lru_isolate_move(lru, &dentry->d_lru, list);
}
@@ -621,6 +647,10 @@
if (dentry->d_op->d_delete(dentry))
return false;
}
+
+ if (unlikely(dentry->d_flags & DCACHE_DONTCACHE))
+ return false;
+
/* retain; LRU fodder */
dentry->d_lockref.count--;
if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST)))
@@ -629,6 +659,21 @@
dentry->d_flags |= DCACHE_REFERENCED;
return true;
}
+
+void d_mark_dontcache(struct inode *inode)
+{
+ struct dentry *de;
+
+ spin_lock(&inode->i_lock);
+ hlist_for_each_entry(de, &inode->i_dentry, d_u.d_alias) {
+ spin_lock(&de->d_lock);
+ de->d_flags |= DCACHE_DONTCACHE;
+ spin_unlock(&de->d_lock);
+ }
+ inode->i_state |= I_DONTCACHE;
+ spin_unlock(&inode->i_lock);
+}
+EXPORT_SYMBOL(d_mark_dontcache);
/*
* Finish off a dentry we've decided to kill.
@@ -835,6 +880,32 @@
}
EXPORT_SYMBOL(dput);
+static void __dput_to_list(struct dentry *dentry, struct list_head *list)
+__must_hold(&dentry->d_lock)
+{
+ if (dentry->d_flags & DCACHE_SHRINK_LIST) {
+ /* let the owner of the list it's on deal with it */
+ --dentry->d_lockref.count;
+ } else {
+ if (dentry->d_flags & DCACHE_LRU_LIST)
+ d_lru_del(dentry);
+ if (!--dentry->d_lockref.count)
+ d_shrink_add(dentry, list);
+ }
+}
+
+void dput_to_list(struct dentry *dentry, struct list_head *list)
+{
+ rcu_read_lock();
+ if (likely(fast_dput(dentry))) {
+ rcu_read_unlock();
+ return;
+ }
+ rcu_read_unlock();
+ if (!retain_dentry(dentry))
+ __dput_to_list(dentry, list);
+ spin_unlock(&dentry->d_lock);
+}
/* This must be called with d_lock held */
static inline void __dget_dlock(struct dentry *dentry)
@@ -1043,7 +1114,7 @@
return false;
}
-static void shrink_dentry_list(struct list_head *list)
+void shrink_dentry_list(struct list_head *list)
{
while (!list_empty(list)) {
struct dentry *dentry, *parent;
@@ -1065,18 +1136,9 @@
rcu_read_unlock();
d_shrink_del(dentry);
parent = dentry->d_parent;
+ if (parent != dentry)
+ __dput_to_list(parent, list);
__dentry_kill(dentry);
- if (parent == dentry)
- continue;
- /*
- * We need to prune ancestors too. This is necessary to prevent
- * quadratic behavior of shrink_dcache_parent(), but is also
- * expected to be beneficial in reducing dentry cache
- * fragmentation.
- */
- dentry = parent;
- while (dentry && !lockref_put_or_lock(&dentry->d_lockref))
- dentry = dentry_kill(dentry);
}
}
@@ -1278,7 +1340,7 @@
if (!list_empty(&dentry->d_subdirs)) {
spin_unlock(&this_parent->d_lock);
- spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_);
+ spin_release(&dentry->d_lock.dep_map, _RET_IP_);
this_parent = dentry;
spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
goto repeat;
@@ -1421,8 +1483,11 @@
struct select_data {
struct dentry *start;
+ union {
+ long found;
+ struct dentry *victim;
+ };
struct list_head dispose;
- int found;
};
static enum d_walk_ret select_collect(void *_data, struct dentry *dentry)
@@ -1454,6 +1519,37 @@
return ret;
}
+static enum d_walk_ret select_collect2(void *_data, struct dentry *dentry)
+{
+ struct select_data *data = _data;
+ enum d_walk_ret ret = D_WALK_CONTINUE;
+
+ if (data->start == dentry)
+ goto out;
+
+ if (dentry->d_flags & DCACHE_SHRINK_LIST) {
+ if (!dentry->d_lockref.count) {
+ rcu_read_lock();
+ data->victim = dentry;
+ return D_WALK_QUIT;
+ }
+ } else {
+ if (dentry->d_flags & DCACHE_LRU_LIST)
+ d_lru_del(dentry);
+ if (!dentry->d_lockref.count)
+ d_shrink_add(dentry, &data->dispose);
+ }
+ /*
+ * We can return to the caller if we have found some (this
+ * ensures forward progress). We'll be coming back to find
+ * the rest.
+ */
+ if (!list_empty(&data->dispose))
+ ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
+out:
+ return ret;
+}
+
/**
* shrink_dcache_parent - prune dcache
* @parent: parent of entries to prune
@@ -1463,12 +1559,9 @@
void shrink_dcache_parent(struct dentry *parent)
{
for (;;) {
- struct select_data data;
+ struct select_data data = {.start = parent};
INIT_LIST_HEAD(&data.dispose);
- data.start = parent;
- data.found = 0;
-
d_walk(parent, &data, select_collect);
if (!list_empty(&data.dispose)) {
@@ -1479,6 +1572,24 @@
cond_resched();
if (!data.found)
break;
+ data.victim = NULL;
+ d_walk(parent, &data, select_collect2);
+ if (data.victim) {
+ struct dentry *parent;
+ spin_lock(&data.victim->d_lock);
+ if (!shrink_lock_dentry(data.victim)) {
+ spin_unlock(&data.victim->d_lock);
+ rcu_read_unlock();
+ } else {
+ rcu_read_unlock();
+ parent = data.victim->d_parent;
+ if (parent != data.victim)
+ __dput_to_list(parent, &data.dispose);
+ __dentry_kill(data.victim);
+ }
+ }
+ if (!list_empty(&data.dispose))
+ shrink_dentry_list(&data.dispose);
}
}
EXPORT_SYMBOL(shrink_dcache_parent);
@@ -1589,7 +1700,7 @@
* copied and the copy passed in may be reused after this call.
*/
-struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
+static struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
{
struct dentry *dentry;
char *dname;
@@ -1635,7 +1746,7 @@
dentry->d_lockref.count = 1;
dentry->d_flags = 0;
spin_lock_init(&dentry->d_lock);
- seqcount_init(&dentry->d_seq);
+ seqcount_spinlock_init(&dentry->d_seq, &dentry->d_lock);
dentry->d_inode = NULL;
dentry->d_parent = dentry;
dentry->d_sb = sb;
@@ -1718,6 +1829,9 @@
* never be anyone's children or parents. Unlike all other
* dentries, these will not have RCU delay between dropping the
* last reference and freeing them.
+ *
+ * The only user is alloc_file_pseudo() and that's what should
+ * be considered a public interface. Don't use directly.
*/
struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name)
{
@@ -1726,7 +1840,6 @@
dentry->d_flags |= DCACHE_NORCU;
return dentry;
}
-EXPORT_SYMBOL(d_alloc_pseudo);
struct dentry *d_alloc_name(struct dentry *parent, const char *name)
{
@@ -1825,6 +1938,11 @@
WARN_ON(d_in_lookup(dentry));
spin_lock(&dentry->d_lock);
+ /*
+ * Decrement negative dentry count if it was in the LRU list.
+ */
+ if (dentry->d_flags & DCACHE_LRU_LIST)
+ this_cpu_dec(nr_dentry_negative);
hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
raw_write_seqcount_begin(&dentry->d_seq);
__d_set_inode_and_type(dentry, inode, add_flags);
@@ -1992,7 +2110,7 @@
{
return __d_obtain_alias(inode, true);
}
-EXPORT_SYMBOL(d_obtain_alias);
+EXPORT_SYMBOL_NS(d_obtain_alias, ANDROID_GKI_VFS_EXPORT_ONLY);
/**
* d_obtain_root - find or allocate a dentry for a given inode
@@ -2066,7 +2184,7 @@
}
return found;
}
-EXPORT_SYMBOL(d_add_ci);
+EXPORT_SYMBOL_NS(d_add_ci, ANDROID_GKI_VFS_EXPORT_ONLY);
static inline bool d_same_name(const struct dentry *dentry,
@@ -2341,7 +2459,6 @@
void d_delete(struct dentry * dentry)
{
struct inode *inode = dentry->d_inode;
- int isdir = d_is_dir(dentry);
spin_lock(&inode->i_lock);
spin_lock(&dentry->d_lock);
@@ -2356,7 +2473,6 @@
spin_unlock(&dentry->d_lock);
spin_unlock(&inode->i_lock);
}
- fsnotify_nameremove(dentry, isdir);
}
EXPORT_SYMBOL(d_delete);
@@ -2949,7 +3065,7 @@
__d_add(dentry, inode);
return NULL;
}
-EXPORT_SYMBOL(d_splice_alias);
+EXPORT_SYMBOL_NS(d_splice_alias, ANDROID_GKI_VFS_EXPORT_ONLY);
/*
* Test whether new_dentry is a subdirectory of old_dentry.
--
Gitblit v1.6.2