Commit b23fb0a6 authored by Nicholas Piggin's avatar Nicholas Piggin
Browse files

fs: scale inode alias list 



Add a new lock, dcache_inode_lock, to protect the inode's i_dentry list
from concurrent modification. d_alias is also protected by d_lock.

Signed-off-by: default avatarNick Piggin <npiggin@kernel.dk>
parent 2fd6b7f5

fs/9p/vfs_inode.c

+2 −0
Original line number Diff line number Diff line
@@ -271,9 +271,11 @@ static struct dentry *v9fs_dentry_from_dir_inode(struct inode *inode) 
	struct dentry *dentry;



	spin_lock(&dcache_lock);

	spin_lock(&dcache_inode_lock);

	/* Directory should have only one entry. */

	BUG_ON(S_ISDIR(inode->i_mode) && !list_is_singular(&inode->i_dentry));

	dentry = list_entry(inode->i_dentry.next, struct dentry, d_alias);

	spin_unlock(&dcache_inode_lock);

	spin_unlock(&dcache_lock);

	return dentry;

}

fs/affs/amigaffs.c

+2 −0
Original line number Diff line number Diff line
@@ -129,6 +129,7 @@ affs_fix_dcache(struct dentry *dentry, u32 entry_ino) 
	struct list_head *head, *next;



	spin_lock(&dcache_lock);

	spin_lock(&dcache_inode_lock);

	head = &inode->i_dentry;

	next = head->next;

	while (next != head) {
@@ -139,6 +140,7 @@ affs_fix_dcache(struct dentry *dentry, u32 entry_ino) 
		}

		next = next->next;

	}

	spin_unlock(&dcache_inode_lock);

	spin_unlock(&dcache_lock);

}

fs/cifs/inode.c

+3 −0
Original line number Diff line number Diff line
@@ -810,12 +810,15 @@ inode_has_hashed_dentries(struct inode *inode) 
	struct dentry *dentry;



	spin_lock(&dcache_lock);

	spin_lock(&dcache_inode_lock);

	list_for_each_entry(dentry, &inode->i_dentry, d_alias) {

		if (!d_unhashed(dentry) || IS_ROOT(dentry)) {

			spin_unlock(&dcache_inode_lock);

			spin_unlock(&dcache_lock);

			return true;

		}

	}

	spin_unlock(&dcache_inode_lock);

	spin_unlock(&dcache_lock);

	return false;

}

fs/dcache.c

+58 −8
Original line number Diff line number Diff line
@@ -37,6 +37,8 @@ 


/*

 * Usage:

 * dcache_inode_lock protects:

 *   - i_dentry, d_alias, d_inode

 * dcache_hash_lock protects:

 *   - the dcache hash table, s_anon lists

 * dcache_lru_lock protects:
@@ -49,9 +51,11 @@ 
 *   - d_unhashed()

 *   - d_parent and d_subdirs

 *   - childrens' d_child and d_parent

 *   - d_alias, d_inode

 *

 * Ordering:

 * dcache_lock

 *   dcache_inode_lock

 *     dentry->d_lock

 *       dcache_lru_lock

 *       dcache_hash_lock
@@ -70,11 +74,13 @@ 
int sysctl_vfs_cache_pressure __read_mostly = 100;

EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);



__cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_inode_lock);

static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_hash_lock);

static __cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lru_lock);

__cacheline_aligned_in_smp DEFINE_SPINLOCK(dcache_lock);

__cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);



EXPORT_SYMBOL(dcache_inode_lock);

EXPORT_SYMBOL(dcache_lock);



static struct kmem_cache *dentry_cache __read_mostly;
@@ -154,6 +160,7 @@ static void d_free(struct dentry *dentry) 
 */

static void dentry_iput(struct dentry * dentry)

	__releases(dentry->d_lock)

	__releases(dcache_inode_lock)

	__releases(dcache_lock)

{

	struct inode *inode = dentry->d_inode;
@@ -161,6 +168,7 @@ static void dentry_iput(struct dentry * dentry) 
		dentry->d_inode = NULL;

		list_del_init(&dentry->d_alias);

		spin_unlock(&dentry->d_lock);

		spin_unlock(&dcache_inode_lock);

		spin_unlock(&dcache_lock);

		if (!inode->i_nlink)

			fsnotify_inoderemove(inode);
@@ -170,6 +178,7 @@ static void dentry_iput(struct dentry * dentry) 
			iput(inode);

	} else {

		spin_unlock(&dentry->d_lock);

		spin_unlock(&dcache_inode_lock);

		spin_unlock(&dcache_lock);

	}

}
@@ -231,6 +240,7 @@ static void dentry_lru_move_tail(struct dentry *dentry) 
static struct dentry *d_kill(struct dentry *dentry, struct dentry *parent)

	__releases(dentry->d_lock)

	__releases(parent->d_lock)

	__releases(dcache_inode_lock)

	__releases(dcache_lock)

{

	list_del(&dentry->d_u.d_child);
@@ -332,13 +342,18 @@ void dput(struct dentry *dentry) 
			 * want to reduce dcache_lock anyway so this will

			 * get improved.

			 */

drop1:

			spin_unlock(&dentry->d_lock);

			goto repeat;

		}

		if (parent && !spin_trylock(&parent->d_lock)) {

			spin_unlock(&dentry->d_lock);

		if (!spin_trylock(&dcache_inode_lock)) {

drop2:

			spin_unlock(&dcache_lock);

			goto repeat;

			goto drop1;

		}

		if (parent && !spin_trylock(&parent->d_lock)) {

			spin_unlock(&dcache_inode_lock);

			goto drop2;

		}

	}

	dentry->d_count--;
@@ -369,6 +384,7 @@ void dput(struct dentry *dentry) 
 	spin_unlock(&dentry->d_lock);

	if (parent)

		spin_unlock(&parent->d_lock);

	spin_unlock(&dcache_inode_lock);

	spin_unlock(&dcache_lock);

	return;
@@ -558,7 +574,9 @@ struct dentry *d_find_alias(struct inode *inode) 


	if (!list_empty(&inode->i_dentry)) {

		spin_lock(&dcache_lock);

		spin_lock(&dcache_inode_lock);

		de = __d_find_alias(inode, 0);

		spin_unlock(&dcache_inode_lock);

		spin_unlock(&dcache_lock);

	}

	return de;
@@ -574,18 +592,21 @@ void d_prune_aliases(struct inode *inode) 
	struct dentry *dentry;

restart:

	spin_lock(&dcache_lock);

	spin_lock(&dcache_inode_lock);

	list_for_each_entry(dentry, &inode->i_dentry, d_alias) {

		spin_lock(&dentry->d_lock);

		if (!dentry->d_count) {

			__dget_locked_dlock(dentry);

			__d_drop(dentry);

			spin_unlock(&dentry->d_lock);

			spin_unlock(&dcache_inode_lock);

			spin_unlock(&dcache_lock);

			dput(dentry);

			goto restart;

		}

		spin_unlock(&dentry->d_lock);

	}

	spin_unlock(&dcache_inode_lock);

	spin_unlock(&dcache_lock);

}

EXPORT_SYMBOL(d_prune_aliases);
@@ -601,6 +622,7 @@ EXPORT_SYMBOL(d_prune_aliases); 
static void prune_one_dentry(struct dentry *dentry, struct dentry *parent)

	__releases(dentry->d_lock)

	__releases(parent->d_lock)

	__releases(dcache_inode_lock)

	__releases(dcache_lock)

{

	__d_drop(dentry);
@@ -612,6 +634,7 @@ static void prune_one_dentry(struct dentry *dentry, struct dentry *parent) 
	 */

	while (dentry) {

		spin_lock(&dcache_lock);

		spin_lock(&dcache_inode_lock);

again:

		spin_lock(&dentry->d_lock);

		if (IS_ROOT(dentry))
@@ -627,6 +650,7 @@ static void prune_one_dentry(struct dentry *dentry, struct dentry *parent) 
			if (parent)

				spin_unlock(&parent->d_lock);

			spin_unlock(&dentry->d_lock);

			spin_unlock(&dcache_inode_lock);

			spin_unlock(&dcache_lock);

			return;

		}
@@ -676,8 +700,9 @@ static void shrink_dentry_list(struct list_head *list) 
		spin_unlock(&dcache_lru_lock);



		prune_one_dentry(dentry, parent);

		/* dcache_lock and dentry->d_lock dropped */

		/* dcache_lock, dcache_inode_lock and dentry->d_lock dropped */

		spin_lock(&dcache_lock);

		spin_lock(&dcache_inode_lock);

		spin_lock(&dcache_lru_lock);

	}

}
@@ -699,6 +724,7 @@ static void __shrink_dcache_sb(struct super_block *sb, int *count, int flags) 
	int cnt = *count;



	spin_lock(&dcache_lock);

	spin_lock(&dcache_inode_lock);

relock:

	spin_lock(&dcache_lru_lock);

	while (!list_empty(&sb->s_dentry_lru)) {
@@ -737,8 +763,8 @@ static void __shrink_dcache_sb(struct super_block *sb, int *count, int flags) 
	if (!list_empty(&referenced))

		list_splice(&referenced, &sb->s_dentry_lru);

	spin_unlock(&dcache_lru_lock);

	spin_unlock(&dcache_inode_lock);

	spin_unlock(&dcache_lock);



}



/**
@@ -832,12 +858,14 @@ void shrink_dcache_sb(struct super_block *sb) 
	LIST_HEAD(tmp);



	spin_lock(&dcache_lock);

	spin_lock(&dcache_inode_lock);

	spin_lock(&dcache_lru_lock);

	while (!list_empty(&sb->s_dentry_lru)) {

		list_splice_init(&sb->s_dentry_lru, &tmp);

		shrink_dentry_list(&tmp);

	}

	spin_unlock(&dcache_lru_lock);

	spin_unlock(&dcache_inode_lock);

	spin_unlock(&dcache_lock);

}

EXPORT_SYMBOL(shrink_dcache_sb);
@@ -1255,9 +1283,11 @@ EXPORT_SYMBOL(d_alloc_name); 
/* the caller must hold dcache_lock */

static void __d_instantiate(struct dentry *dentry, struct inode *inode)

{

	spin_lock(&dentry->d_lock);

	if (inode)

		list_add(&dentry->d_alias, &inode->i_dentry);

	dentry->d_inode = inode;

	spin_unlock(&dentry->d_lock);

	fsnotify_d_instantiate(dentry, inode);

}
@@ -1280,7 +1310,9 @@ void d_instantiate(struct dentry *entry, struct inode * inode) 
{

	BUG_ON(!list_empty(&entry->d_alias));

	spin_lock(&dcache_lock);

	spin_lock(&dcache_inode_lock);

	__d_instantiate(entry, inode);

	spin_unlock(&dcache_inode_lock);

	spin_unlock(&dcache_lock);

	security_d_instantiate(entry, inode);

}
@@ -1341,7 +1373,9 @@ struct dentry *d_instantiate_unique(struct dentry *entry, struct inode *inode) 
	BUG_ON(!list_empty(&entry->d_alias));



	spin_lock(&dcache_lock);

	spin_lock(&dcache_inode_lock);

	result = __d_instantiate_unique(entry, inode);

	spin_unlock(&dcache_inode_lock);

	spin_unlock(&dcache_lock);



	if (!result) {
@@ -1432,8 +1466,10 @@ struct dentry *d_obtain_alias(struct inode *inode) 
	tmp->d_parent = tmp; /* make sure dput doesn't croak */



	spin_lock(&dcache_lock);

	spin_lock(&dcache_inode_lock);

	res = __d_find_alias(inode, 0);

	if (res) {

		spin_unlock(&dcache_inode_lock);

		spin_unlock(&dcache_lock);

		dput(tmp);

		goto out_iput;
@@ -1450,6 +1486,7 @@ struct dentry *d_obtain_alias(struct inode *inode) 
	hlist_add_head(&tmp->d_hash, &inode->i_sb->s_anon);

	spin_unlock(&dcache_hash_lock);

	spin_unlock(&tmp->d_lock);

	spin_unlock(&dcache_inode_lock);



	spin_unlock(&dcache_lock);

	return tmp;
@@ -1482,9 +1519,11 @@ struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry) 


	if (inode && S_ISDIR(inode->i_mode)) {

		spin_lock(&dcache_lock);

		spin_lock(&dcache_inode_lock);

		new = __d_find_alias(inode, 1);

		if (new) {

			BUG_ON(!(new->d_flags & DCACHE_DISCONNECTED));

			spin_unlock(&dcache_inode_lock);

			spin_unlock(&dcache_lock);

			security_d_instantiate(new, inode);

			d_move(new, dentry);
@@ -1492,6 +1531,7 @@ struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry) 
		} else {

			/* already taking dcache_lock, so d_add() by hand */

			__d_instantiate(dentry, inode);

			spin_unlock(&dcache_inode_lock);

			spin_unlock(&dcache_lock);

			security_d_instantiate(dentry, inode);

			d_rehash(dentry);
@@ -1566,8 +1606,10 @@ struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode, 
	 * already has a dentry.

	 */

	spin_lock(&dcache_lock);

	spin_lock(&dcache_inode_lock);

	if (!S_ISDIR(inode->i_mode) || list_empty(&inode->i_dentry)) {

		__d_instantiate(found, inode);

		spin_unlock(&dcache_inode_lock);

		spin_unlock(&dcache_lock);

		security_d_instantiate(found, inode);

		return found;
@@ -1579,6 +1621,7 @@ struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode, 
	 */

	new = list_entry(inode->i_dentry.next, struct dentry, d_alias);

	dget_locked(new);

	spin_unlock(&dcache_inode_lock);

	spin_unlock(&dcache_lock);

	security_d_instantiate(found, inode);

	d_move(new, found);
@@ -1797,6 +1840,7 @@ void d_delete(struct dentry * dentry) 
	 * Are we the only user?

	 */

	spin_lock(&dcache_lock);

	spin_lock(&dcache_inode_lock);

	spin_lock(&dentry->d_lock);

	isdir = S_ISDIR(dentry->d_inode->i_mode);

	if (dentry->d_count == 1) {
@@ -1810,6 +1854,7 @@ void d_delete(struct dentry * dentry) 
		__d_drop(dentry);



	spin_unlock(&dentry->d_lock);

	spin_unlock(&dcache_inode_lock);

	spin_unlock(&dcache_lock);



	fsnotify_nameremove(dentry, isdir);
@@ -2067,6 +2112,7 @@ struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2) 
 */

static struct dentry *__d_unalias(struct dentry *dentry, struct dentry *alias)

	__releases(dcache_lock)

	__releases(dcache_inode_lock)

{

	struct mutex *m1 = NULL, *m2 = NULL;

	struct dentry *ret;
@@ -2092,6 +2138,7 @@ static struct dentry *__d_unalias(struct dentry *dentry, struct dentry *alias) 
	d_move_locked(alias, dentry);

	ret = alias;

out_err:

	spin_unlock(&dcache_inode_lock);

	spin_unlock(&dcache_lock);

	if (m2)

		mutex_unlock(m2);
@@ -2153,6 +2200,7 @@ struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode) 
	BUG_ON(!d_unhashed(dentry));



	spin_lock(&dcache_lock);

	spin_lock(&dcache_inode_lock);



	if (!inode) {

		actual = dentry;
@@ -2196,6 +2244,7 @@ struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode) 
	_d_rehash(actual);

	spin_unlock(&dcache_hash_lock);

	spin_unlock(&actual->d_lock);

	spin_unlock(&dcache_inode_lock);

	spin_unlock(&dcache_lock);

out_nolock:

	if (actual == dentry) {
@@ -2207,6 +2256,7 @@ struct dentry *d_materialise_unique(struct dentry *dentry, struct inode *inode) 
	return actual;



shouldnt_be_hashed:

	spin_unlock(&dcache_inode_lock);

	spin_unlock(&dcache_lock);

	BUG();

}

fs/exportfs/expfs.c

+4 −0
Original line number Diff line number Diff line
@@ -48,8 +48,10 @@ find_acceptable_alias(struct dentry *result, 
		return result;



	spin_lock(&dcache_lock);

	spin_lock(&dcache_inode_lock);

	list_for_each_entry(dentry, &result->d_inode->i_dentry, d_alias) {

		dget_locked(dentry);

		spin_unlock(&dcache_inode_lock);

		spin_unlock(&dcache_lock);

		if (toput)

			dput(toput);
@@ -58,8 +60,10 @@ find_acceptable_alias(struct dentry *result, 
			return dentry;

		}

		spin_lock(&dcache_lock);

		spin_lock(&dcache_inode_lock);

		toput = dentry;

	}

	spin_unlock(&dcache_inode_lock);

	spin_unlock(&dcache_lock);



	if (toput)