Commit d9ef5a8c authored by Chuck Lever's avatar Chuck Lever Committed by Trond Myklebust
Browse files

NFS: introduce mechanism for tracking NFS client metrics 



Add a per-superblock performance counter facility to the NFS client.  This
facility mimics the counters available for block devices and for
networking.  Expose these new counters via the new /proc/self/mountstats
interface.

Thanks to Andrew Morton and Trond Myklebust for their review and comments.

Test plan:
fsx and iozone on UP and SMP systems, with and without pre-emption.  Watch
for memory overwrite bugs, and performance loss (significantly more CPU
required per op).

Signed-off-by: default avatarChuck Lever <cel@netapp.com>
Signed-off-by: default avatarTrond Myklebust <Trond.Myklebust@netapp.com>
parent c8bded96

fs/nfs/inode.c

+97 −6
Original line number Diff line number Diff line
@@ -42,6 +42,7 @@ 
#include "nfs4_fs.h"

#include "callback.h"

#include "delegation.h"

#include "iostat.h"



#define NFSDBG_FACILITY		NFSDBG_VFS

#define NFS_PARANOIA 1
@@ -65,6 +66,7 @@ static void nfs_clear_inode(struct inode *); 
static void nfs_umount_begin(struct super_block *);

static int  nfs_statfs(struct super_block *, struct kstatfs *);

static int  nfs_show_options(struct seq_file *, struct vfsmount *);

static int  nfs_show_stats(struct seq_file *, struct vfsmount *);

static void nfs_zap_acl_cache(struct inode *);



static struct rpc_program	nfs_program;
@@ -78,6 +80,7 @@ static struct super_operations nfs_sops = { 
	.clear_inode	= nfs_clear_inode,

	.umount_begin	= nfs_umount_begin,

	.show_options	= nfs_show_options,

	.show_stats	= nfs_show_stats,

};



/*
@@ -290,6 +293,12 @@ nfs_sb_init(struct super_block *sb, rpc_authflavor_t authflavor) 
	}

	sb->s_root->d_op = server->rpc_ops->dentry_ops;



	server->io_stats = nfs_alloc_iostats();

	if (!server->io_stats) {

		no_root_error = -ENOMEM;

		goto out_no_root;

	}



	/* Get some general file system info */

	if (server->namelen == 0 &&

	    server->rpc_ops->pathconf(server, &server->fh, &pathinfo) >= 0)
@@ -582,7 +591,7 @@ nfs_statfs(struct super_block *sb, struct kstatfs *buf) 


}



static int nfs_show_options(struct seq_file *m, struct vfsmount *mnt)

static void nfs_show_mount_options(struct seq_file *m, struct nfs_server *nfss, int showdefaults)

{

	static struct proc_nfs_info {

		int flag;
@@ -598,20 +607,19 @@ static int nfs_show_options(struct seq_file *m, struct vfsmount *mnt) 
		{ 0, NULL, NULL }

	};

	struct proc_nfs_info *nfs_infop;

	struct nfs_server *nfss = NFS_SB(mnt->mnt_sb);

	char buf[12];

	char *proto;



	seq_printf(m, ",vers=%d", nfss->rpc_ops->version);

	seq_printf(m, ",rsize=%d", nfss->rsize);

	seq_printf(m, ",wsize=%d", nfss->wsize);

	if (nfss->acregmin != 3*HZ)

	if (nfss->acregmin != 3*HZ || showdefaults)

		seq_printf(m, ",acregmin=%d", nfss->acregmin/HZ);

	if (nfss->acregmax != 60*HZ)

	if (nfss->acregmax != 60*HZ || showdefaults)

		seq_printf(m, ",acregmax=%d", nfss->acregmax/HZ);

	if (nfss->acdirmin != 30*HZ)

	if (nfss->acdirmin != 30*HZ || showdefaults)

		seq_printf(m, ",acdirmin=%d", nfss->acdirmin/HZ);

	if (nfss->acdirmax != 60*HZ)

	if (nfss->acdirmax != 60*HZ || showdefaults)

		seq_printf(m, ",acdirmax=%d", nfss->acdirmax/HZ);

	for (nfs_infop = nfs_info; nfs_infop->flag; nfs_infop++) {

		if (nfss->flags & nfs_infop->flag)
@@ -633,8 +641,89 @@ static int nfs_show_options(struct seq_file *m, struct vfsmount *mnt) 
	seq_printf(m, ",proto=%s", proto);

	seq_printf(m, ",timeo=%lu", 10U * nfss->retrans_timeo / HZ);

	seq_printf(m, ",retrans=%u", nfss->retrans_count);

}



static int nfs_show_options(struct seq_file *m, struct vfsmount *mnt)

{

	struct nfs_server *nfss = NFS_SB(mnt->mnt_sb);



	nfs_show_mount_options(m, nfss, 0);



	seq_puts(m, ",addr=");

	seq_escape(m, nfss->hostname, " \t\n\\");



	return 0;

}



static int nfs_show_stats(struct seq_file *m, struct vfsmount *mnt)

{

	int i, cpu;

	struct nfs_server *nfss = NFS_SB(mnt->mnt_sb);

	struct rpc_auth *auth = nfss->client->cl_auth;

	struct nfs_iostats totals = { };



	seq_printf(m, "statvers=%s", NFS_IOSTAT_VERS);



	/*

	 * Display all mount option settings

	 */

	seq_printf(m, "\n\topts:\t");

	seq_puts(m, mnt->mnt_sb->s_flags & MS_RDONLY ? "ro" : "rw");

	seq_puts(m, mnt->mnt_sb->s_flags & MS_SYNCHRONOUS ? ",sync" : "");

	seq_puts(m, mnt->mnt_sb->s_flags & MS_NOATIME ? ",noatime" : "");

	seq_puts(m, mnt->mnt_sb->s_flags & MS_NODIRATIME ? ",nodiratime" : "");

	nfs_show_mount_options(m, nfss, 1);



	seq_printf(m, "\n\tcaps:\t");

	seq_printf(m, "caps=0x%x", nfss->caps);

	seq_printf(m, ",wtmult=%d", nfss->wtmult);

	seq_printf(m, ",dtsize=%d", nfss->dtsize);

	seq_printf(m, ",bsize=%d", nfss->bsize);

	seq_printf(m, ",namelen=%d", nfss->namelen);



#ifdef CONFIG_NFS_V4

	if (nfss->rpc_ops->version == 4) {

		seq_printf(m, "\n\tnfsv4:\t");

		seq_printf(m, "bm0=0x%x", nfss->attr_bitmask[0]);

		seq_printf(m, ",bm1=0x%x", nfss->attr_bitmask[1]);

		seq_printf(m, ",acl=0x%x", nfss->acl_bitmask);

	}

#endif



	/*

	 * Display security flavor in effect for this mount

	 */

	seq_printf(m, "\n\tsec:\tflavor=%d", auth->au_ops->au_flavor);

	if (auth->au_flavor)

		seq_printf(m, ",pseudoflavor=%d", auth->au_flavor);



	/*

	 * Display superblock I/O counters

	 */

	for (cpu = 0; cpu < NR_CPUS; cpu++) {

		struct nfs_iostats *stats;



		if (!cpu_possible(cpu))

			continue;



		preempt_disable();

		stats = per_cpu_ptr(nfss->io_stats, cpu);



		for (i = 0; i < __NFSIOS_COUNTSMAX; i++)

			totals.events[i] += stats->events[i];

		for (i = 0; i < __NFSIOS_BYTESMAX; i++)

			totals.bytes[i] += stats->bytes[i];



		preempt_enable();

	}



	seq_printf(m, "\n\tevents:\t");

	for (i = 0; i < __NFSIOS_COUNTSMAX; i++)

		seq_printf(m, "%lu ", totals.events[i]);

	seq_printf(m, "\n\tbytes:\t");

	for (i = 0; i < __NFSIOS_BYTESMAX; i++)

		seq_printf(m, "%Lu ", totals.bytes[i]);



	return 0;

}
@@ -1742,6 +1831,7 @@ static struct super_operations nfs4_sops = { 
	.clear_inode	= nfs4_clear_inode,

	.umount_begin	= nfs_umount_begin,

	.show_options	= nfs_show_options,

	.show_stats	= nfs_show_stats,

};



/*
@@ -2015,6 +2105,7 @@ static struct super_block *nfs4_get_sb(struct file_system_type *fs_type, 
out_free:

	kfree(server->mnt_path);

	kfree(server->hostname);

	nfs_free_iostats(server->io_stats);

	kfree(server);

	return s;

}

fs/nfs/iostat.h

0 → 100644
+152 −0
Original line number Diff line number Diff line 
/*

 *  linux/fs/nfs/iostat.h

 *

 *  Declarations for NFS client per-mount statistics

 *

 *  Copyright (C) 2005, 2006 Chuck Lever <cel@netapp.com>

 *

 *  NFS client per-mount statistics provide information about the health of

 *  the NFS client and the health of each NFS mount point.  Generally these

 *  are not for detailed problem diagnosis, but simply to indicate that there

 *  is a problem.

 *

 *  These counters are not meant to be human-readable, but are meant to be

 *  integrated into system monitoring tools such as "sar" and "iostat".  As

 *  such, the counters are sampled by the tools over time, and are never

 *  zeroed after a file system is mounted.  Moving averages can be computed

 *  by the tools by taking the difference between two instantaneous samples

 *  and dividing that by the time between the samples.

 */



#ifndef _NFS_IOSTAT

#define _NFS_IOSTAT



#define NFS_IOSTAT_VERS		"1.0"



/*

 * NFS byte counters

 *

 * 1.  SERVER - the number of payload bytes read from or written to the

 *     server by the NFS client via an NFS READ or WRITE request.

 *

 * 2.  NORMAL - the number of bytes read or written by applications via

 *     the read(2) and write(2) system call interfaces.

 *

 * 3.  DIRECT - the number of bytes read or written from files opened

 *     with the O_DIRECT flag.

 *

 * These counters give a view of the data throughput into and out of the NFS

 * client.  Comparing the number of bytes requested by an application with the

 * number of bytes the client requests from the server can provide an

 * indication of client efficiency (per-op, cache hits, etc).

 *

 * These counters can also help characterize which access methods are in

 * use.  DIRECT by itself shows whether there is any O_DIRECT traffic.

 * NORMAL + DIRECT shows how much data is going through the system call

 * interface.  A large amount of SERVER traffic without much NORMAL or

 * DIRECT traffic shows that applications are using mapped files.

 *

 * NFS page counters

 *

 * These count the number of pages read or written via nfs_readpage(),

 * nfs_readpages(), or their write equivalents.

 */

enum nfs_stat_bytecounters {

	NFSIOS_NORMALREADBYTES = 0,

	NFSIOS_NORMALWRITTENBYTES,

	NFSIOS_DIRECTREADBYTES,

	NFSIOS_DIRECTWRITTENBYTES,

	NFSIOS_SERVERREADBYTES,

	NFSIOS_SERVERWRITTENBYTES,

	NFSIOS_READPAGES,

	NFSIOS_WRITEPAGES,

	__NFSIOS_BYTESMAX,

};



/*

 * NFS event counters

 *

 * These counters provide a low-overhead way of monitoring client activity

 * without enabling NFS trace debugging.  The counters show the rate at

 * which VFS requests are made, and how often the client invalidates its

 * data and attribute caches.  This allows system administrators to monitor

 * such things as how close-to-open is working, and answer questions such

 * as "why are there so many GETATTR requests on the wire?"

 *

 * They also count anamolous events such as short reads and writes, silly

 * renames due to close-after-delete, and operations that change the size

 * of a file (such operations can often be the source of data corruption

 * if applications aren't using file locking properly).

 */

enum nfs_stat_eventcounters {

	NFSIOS_INODEREVALIDATE = 0,

	NFSIOS_DENTRYREVALIDATE,

	NFSIOS_DATAINVALIDATE,

	NFSIOS_ATTRINVALIDATE,

	NFSIOS_VFSOPEN,

	NFSIOS_VFSLOOKUP,

	NFSIOS_VFSACCESS,

	NFSIOS_VFSUPDATEPAGE,

	NFSIOS_VFSREADPAGE,

	NFSIOS_VFSREADPAGES,

	NFSIOS_VFSWRITEPAGE,

	NFSIOS_VFSWRITEPAGES,

	NFSIOS_VFSGETDENTS,

	NFSIOS_VFSSETATTR,

	NFSIOS_VFSFLUSH,

	NFSIOS_VFSFSYNC,

	NFSIOS_VFSLOCK,

	NFSIOS_VFSRELEASE,

	NFSIOS_CONGESTIONWAIT,

	NFSIOS_SETATTRTRUNC,

	NFSIOS_EXTENDWRITE,

	NFSIOS_SILLYRENAME,

	NFSIOS_SHORTREAD,

	NFSIOS_SHORTWRITE,

	__NFSIOS_COUNTSMAX,

};



#ifdef __KERNEL__



#include <linux/percpu.h>

#include <linux/cache.h>



struct nfs_iostats {

	unsigned long long	bytes[__NFSIOS_BYTESMAX];

	unsigned long		events[__NFSIOS_COUNTSMAX];

} ____cacheline_aligned;



static inline void nfs_inc_stats(struct inode *inode, enum nfs_stat_eventcounters stat)

{

	struct nfs_iostats *iostats;

	int cpu;



	cpu = get_cpu();

	iostats = per_cpu_ptr(NFS_SERVER(inode)->io_stats, cpu);

	iostats->events[stat] ++;

	put_cpu_no_resched();

}



static inline void nfs_add_stats(struct inode *inode, enum nfs_stat_bytecounters stat, unsigned long addend)

{

	struct nfs_iostats *iostats;

	int cpu;



	cpu = get_cpu();

	iostats = per_cpu_ptr(NFS_SERVER(inode)->io_stats, cpu);

	iostats->bytes[stat] += addend;

	put_cpu_no_resched();

}



static inline struct nfs_iostats *nfs_alloc_iostats(void)

{

	return alloc_percpu(struct nfs_iostats);

}



static inline void nfs_free_iostats(struct nfs_iostats *stats)

{

	free_percpu(stats);

}



#endif

#endif

include/linux/nfs_fs_sb.h

+3 −0
Original line number Diff line number Diff line
@@ -4,6 +4,8 @@ 
#include <linux/list.h>

#include <linux/backing-dev.h>



struct nfs_iostats;



/*

 * NFS client parameters stored in the superblock.

 */
@@ -12,6 +14,7 @@ struct nfs_server { 
	struct rpc_clnt *	client_sys;	/* 2nd handle for FSINFO */

	struct rpc_clnt *	client_acl;	/* ACL RPC client handle */

	struct nfs_rpc_ops *	rpc_ops;	/* NFS protocol vector */

	struct nfs_iostats *	io_stats;	/* I/O statistics */

	struct backing_dev_info	backing_dev_info;

	int			flags;		/* various flags */

	unsigned int		caps;		/* server capabilities */