Skip to content

Commit

Permalink
dcache: remove dentries from LRU before putting on dispose list
Browse files Browse the repository at this point in the history 
One of the big problems with modifying the way the dcache shrinker and LRU
implementation works is that the LRU is abused in several ways.  One of
these is shrink_dentry_list().

Basically, we can move a dentry off the LRU onto a different list without
doing any accounting changes, and then use dentry_lru_prune() to remove it
from what-ever list it is now on to do the LRU accounting at that point.

This makes it -really hard- to change the LRU implementation.  The use of
the per-sb LRU lock serialises movement of the dentries between the
different lists and the removal of them, and this is the only reason that
it works.  If we want to break up the dentry LRU lock and lists into, say,
per-node lists, we remove the only serialisation that allows this lru
list/dispose list abuse to work.

To make this work effectively, the dispose list has to be isolated from
the LRU list - dentries have to be removed from the LRU *before* being
placed on the dispose list.  This means that the LRU accounting and
isolation is completed before disposal is started, and that means we can
change the LRU implementation freely in future.

This means that dentries *must* be marked with DCACHE_SHRINK_LIST when
they are placed on the dispose list so that we don't think that parent
dentries found in try_prune_one_dentry() are on the LRU when the are
actually on the dispose list.  This would result in accounting the dentry
to the LRU a second time.  Hence dentry_lru_del() has to handle the
DCACHE_SHRINK_LIST case

Signed-off-by: Dave Chinner <[email protected]>
Signed-off-by: Glauber Costa <[email protected]>
Cc: "Theodore Ts'o" <[email protected]>
Cc: Adrian Hunter <[email protected]>
Cc: Al Viro <[email protected]>
Cc: Artem Bityutskiy <[email protected]>
Cc: Arve Hjønnevåg <[email protected]>
Cc: Carlos Maiolino <[email protected]>
Cc: Christoph Hellwig <[email protected]>
Cc: Chuck Lever <[email protected]>
Cc: Daniel Vetter <[email protected]>
Cc: David Rientjes <[email protected]>
Cc: Gleb Natapov <[email protected]>
Cc: Greg Thelen <[email protected]>
Cc: J. Bruce Fields <[email protected]>
Cc: Jan Kara <[email protected]>
Cc: Jerome Glisse <[email protected]>
Cc: John Stultz <[email protected]>
Cc: KAMEZAWA Hiroyuki <[email protected]>
Cc: Kent Overstreet <[email protected]>
Cc: Kirill A. Shutemov <[email protected]>
Cc: Marcelo Tosatti <[email protected]>
Cc: Mel Gorman <[email protected]>
Cc: Steven Whitehouse <[email protected]>
Cc: Thomas Hellstrom <[email protected]>
Cc: Trond Myklebust <[email protected]>
Signed-off-by: Andrew Morton <[email protected]>
Signed-off-by: Al Viro <[email protected]>
  • Loading branch information
Dave Chinner authored and Al Viro committed Sep 10, 2013
1 parent 1915684 commit dd1f6b2
Showing 1 changed file with 78 additions and 21 deletions.
99 changes: 78 additions & 21 deletions fs/dcache.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ static void dentry_unlink_inode(struct dentry * dentry)
}

/*
* dentry_lru_(add|del|prune|move_tail) must be called with d_lock held.
* dentry_lru_(add|del|move_list) must be called with d_lock held.
*/
static void dentry_lru_add(struct dentry *dentry)
{
Expand All @@ -373,16 +373,26 @@ static void dentry_lru_add(struct dentry *dentry)
static void __dentry_lru_del(struct dentry *dentry)
{
list_del_init(&dentry->d_lru);
dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
dentry->d_flags &= ~DCACHE_LRU_LIST;
dentry->d_sb->s_nr_dentry_unused--;
this_cpu_dec(nr_dentry_unused);
}

/*
* Remove a dentry with references from the LRU.
*
* If we are on the shrink list, then we can get to try_prune_one_dentry() and
* lose our last reference through the parent walk. In this case, we need to
* remove ourselves from the shrink list, not the LRU.
*/
static void dentry_lru_del(struct dentry *dentry)
{
if (dentry->d_flags & DCACHE_SHRINK_LIST) {
list_del_init(&dentry->d_lru);
dentry->d_flags &= ~DCACHE_SHRINK_LIST;
return;
}

if (!list_empty(&dentry->d_lru)) {
spin_lock(&dentry->d_sb->s_dentry_lru_lock);
__dentry_lru_del(dentry);
Expand All @@ -392,14 +402,16 @@ static void dentry_lru_del(struct dentry *dentry)

static void dentry_lru_move_list(struct dentry *dentry, struct list_head *list)
{
BUG_ON(dentry->d_flags & DCACHE_SHRINK_LIST);

spin_lock(&dentry->d_sb->s_dentry_lru_lock);
if (list_empty(&dentry->d_lru)) {
dentry->d_flags |= DCACHE_LRU_LIST;
list_add_tail(&dentry->d_lru, list);
dentry->d_sb->s_nr_dentry_unused++;
this_cpu_inc(nr_dentry_unused);
} else {
list_move_tail(&dentry->d_lru, list);
dentry->d_sb->s_nr_dentry_unused--;
this_cpu_dec(nr_dentry_unused);
}
spin_unlock(&dentry->d_sb->s_dentry_lru_lock);
}
Expand Down Expand Up @@ -497,7 +509,8 @@ EXPORT_SYMBOL(d_drop);
* If ref is non-zero, then decrement the refcount too.
* Returns dentry requiring refcount drop, or NULL if we're done.
*/
static inline struct dentry *dentry_kill(struct dentry *dentry)
static inline struct dentry *
dentry_kill(struct dentry *dentry, int unlock_on_failure)
__releases(dentry->d_lock)
{
struct inode *inode;
Expand All @@ -506,8 +519,10 @@ static inline struct dentry *dentry_kill(struct dentry *dentry)
inode = dentry->d_inode;
if (inode && !spin_trylock(&inode->i_lock)) {
relock:
spin_unlock(&dentry->d_lock);
cpu_relax();
if (unlock_on_failure) {
spin_unlock(&dentry->d_lock);
cpu_relax();
}
return dentry; /* try again with same dentry */
}
if (IS_ROOT(dentry))
Expand Down Expand Up @@ -590,7 +605,7 @@ void dput(struct dentry *dentry)
return;

kill_it:
dentry = dentry_kill(dentry);
dentry = dentry_kill(dentry, 1);
if (dentry)
goto repeat;
}
Expand Down Expand Up @@ -810,12 +825,12 @@ EXPORT_SYMBOL(d_prune_aliases);
*
* This may fail if locks cannot be acquired no problem, just try again.
*/
static void try_prune_one_dentry(struct dentry *dentry)
static struct dentry * try_prune_one_dentry(struct dentry *dentry)
__releases(dentry->d_lock)
{
struct dentry *parent;

parent = dentry_kill(dentry);
parent = dentry_kill(dentry, 0);
/*
* If dentry_kill returns NULL, we have nothing more to do.
* if it returns the same dentry, trylocks failed. In either
Expand All @@ -827,17 +842,18 @@ static void try_prune_one_dentry(struct dentry *dentry)
* fragmentation.
*/
if (!parent)
return;
return NULL;
if (parent == dentry)
return;
return dentry;

/* Prune ancestors. */
dentry = parent;
while (dentry) {
if (lockref_put_or_lock(&dentry->d_lockref))
return;
dentry = dentry_kill(dentry);
return NULL;
dentry = dentry_kill(dentry, 1);
}
return NULL;
}

static void shrink_dentry_list(struct list_head *list)
Expand All @@ -855,22 +871,32 @@ static void shrink_dentry_list(struct list_head *list)
continue;
}

/*
* The dispose list is isolated and dentries are not accounted
* to the LRU here, so we can simply remove it from the list
* here regardless of whether it is referenced or not.
*/
list_del_init(&dentry->d_lru);
dentry->d_flags &= ~DCACHE_SHRINK_LIST;

/*
* We found an inuse dentry which was not removed from
* the LRU because of laziness during lookup. Do not free
* it - just keep it off the LRU list.
* the LRU because of laziness during lookup. Do not free it.
*/
if (dentry->d_lockref.count) {
dentry_lru_del(dentry);
spin_unlock(&dentry->d_lock);
continue;
}

rcu_read_unlock();

try_prune_one_dentry(dentry);
dentry = try_prune_one_dentry(dentry);

rcu_read_lock();
if (dentry) {
dentry->d_flags |= DCACHE_SHRINK_LIST;
list_add(&dentry->d_lru, list);
spin_unlock(&dentry->d_lock);
}
}
rcu_read_unlock();
}
Expand Down Expand Up @@ -911,8 +937,10 @@ void prune_dcache_sb(struct super_block *sb, int count)
list_move(&dentry->d_lru, &referenced);
spin_unlock(&dentry->d_lock);
} else {
list_move_tail(&dentry->d_lru, &tmp);
list_move(&dentry->d_lru, &tmp);
dentry->d_flags |= DCACHE_SHRINK_LIST;
this_cpu_dec(nr_dentry_unused);
sb->s_nr_dentry_unused--;
spin_unlock(&dentry->d_lock);
if (!--count)
break;
Expand All @@ -926,6 +954,27 @@ void prune_dcache_sb(struct super_block *sb, int count)
shrink_dentry_list(&tmp);
}

/*
* Mark all the dentries as on being the dispose list so we don't think they are
* still on the LRU if we try to kill them from ascending the parent chain in
* try_prune_one_dentry() rather than directly from the dispose list.
*/
static void
shrink_dcache_list(
struct list_head *dispose)
{
struct dentry *dentry;

rcu_read_lock();
list_for_each_entry_rcu(dentry, dispose, d_lru) {
spin_lock(&dentry->d_lock);
dentry->d_flags |= DCACHE_SHRINK_LIST;
spin_unlock(&dentry->d_lock);
}
rcu_read_unlock();
shrink_dentry_list(dispose);
}

/**
* shrink_dcache_sb - shrink dcache for a superblock
* @sb: superblock
Expand All @@ -939,9 +988,17 @@ void shrink_dcache_sb(struct super_block *sb)

spin_lock(&sb->s_dentry_lru_lock);
while (!list_empty(&sb->s_dentry_lru)) {
/*
* account for removal here so we don't need to handle it later
* even though the dentry is no longer on the lru list.
*/
list_splice_init(&sb->s_dentry_lru, &tmp);
this_cpu_sub(nr_dentry_unused, sb->s_nr_dentry_unused);
sb->s_nr_dentry_unused = 0;
spin_unlock(&sb->s_dentry_lru_lock);
shrink_dentry_list(&tmp);

shrink_dcache_list(&tmp);

spin_lock(&sb->s_dentry_lru_lock);
}
spin_unlock(&sb->s_dentry_lru_lock);
Expand Down

0 comments on commit dd1f6b2

Please sign in to comment.