1#@!#!123s

: / usr / sbin /
Filename : mountstats
back
#!/usr/bin/python3
# -*- python-mode -*-
"""Parse /proc/self/mountstats and display it in human readable form
"""

from __future__ import print_function
import datetime as datetime

__copyright__ = """
Copyright (C) 2005, Chuck Lever <cel@netapp.com>

This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License version 2 as
published by the Free Software Foundation.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
MA 02110-1301 USA
"""

import sys, os, time
from operator import itemgetter, add
try:
    import argparse
except ImportError:
    print('%s:  Failed to import argparse - make sure argparse is installed!'
        % sys.argv[0])
    sys.exit(1)

Mountstats_version = '0.3'

def difference(x, y):
    """Used for a map() function
    """
    return x - y

NfsEventCounters = [
    'inoderevalidates',
    'dentryrevalidates',
    'datainvalidates',
    'attrinvalidates',
    'vfsopen',
    'vfslookup',
    'vfspermission',
    'vfsupdatepage',
    'vfsreadpage',
    'vfsreadpages',
    'vfswritepage',
    'vfswritepages',
    'vfsreaddir',
    'vfssetattr',
    'vfsflush',
    'vfsfsync',
    'vfslock',
    'vfsrelease',
    'congestionwait',
    'setattrtrunc',
    'extendwrite',
    'sillyrenames',
    'shortreads',
    'shortwrites',
    'delay',
    'pnfsreads',
    'pnfswrites'
]

NfsByteCounters = [
    'normalreadbytes',
    'normalwritebytes',
    'directreadbytes',
    'directwritebytes',
    'serverreadbytes',
    'serverwritebytes',
    'readpages',
    'writepages'
]

XprtUdpCounters = [
    'port',
    'bind_count',
    'rpcsends',
    'rpcreceives',
    'badxids',
    'inflightsends',
    'backlogutil',
    'maxslots',
    'sendutil',
    'pendutil'
]

XprtTcpCounters = [
    'port',
    'bind_count',
    'connect_count',
    'connect_time',
    'idle_time',
    'rpcsends',
    'rpcreceives',
    'badxids',
    'inflightsends',
    'backlogutil',
    'maxslots',
    'sendutil',
    'pendutil'
]

XprtRdmaCounters = [
    'port',
    'bind_count',
    'connect_count',
    'connect_time',
    'idle_time',
    'rpcsends',
    'rpcreceives',
    'badxids',
    'inflightsends',
    'backlogutil',
    'read_segments',
    'write_segments',
    'reply_segments',
    'total_rdma_req',
    'total_rdma_rep',
    'pullup',
    'fixup',
    'hardway',
    'failed_marshal',
    'bad_reply',
    'nomsg_calls',
    'recovered_mrs',
    'orphaned_mrs',
    'allocated_mrs',
    'local_invalidates',
    'empty_sendctx_q',
    'reply_waits_for_send',
]

Nfsv3ops = [
    'NULL',
    'GETATTR',
    'SETATTR',
    'LOOKUP',
    'ACCESS',
    'READLINK',
    'READ',
    'WRITE',
    'CREATE',
    'MKDIR',
    'SYMLINK',
    'MKNOD',
    'REMOVE',
    'RMDIR',
    'RENAME',
    'LINK',
    'READDIR',
    'READDIRPLUS',
    'FSSTAT',
    'FSINFO',
    'PATHCONF',
    'COMMIT'
]

# This list should be kept in-sync with the NFSPROC4_CLNT_* enum in
# include/linux/nfs4.h in the kernel.
Nfsv4ops = [
    'NULL',
    'READ',
    'WRITE',
    'COMMIT',
    'OPEN',
    'OPEN_CONFIRM',
    'OPEN_NOATTR',
    'OPEN_DOWNGRADE',
    'CLOSE',
    'SETATTR',
    'FSINFO',
    'RENEW',
    'SETCLIENTID',
    'SETCLIENTID_CONFIRM',
    'LOCK',
    'LOCKT',
    'LOCKU',
    'ACCESS',
    'GETATTR',
    'LOOKUP',
    'LOOKUP_ROOT',
    'REMOVE',
    'RENAME',
    'LINK',
    'SYMLINK',
    'CREATE',
    'PATHCONF',
    'STATFS',
    'READLINK',
    'READDIR',
    'SERVER_CAPS',
    'DELEGRETURN',
    'GETACL',
    'SETACL',
    'FS_LOCATIONS',
    'RELEASE_LOCKOWNER',
    'SECINFO',
    'FSID_PRESENT',
    'EXCHANGE_ID',
    'CREATE_SESSION',
    'DESTROY_SESSION',
    'SEQUENCE',
    'GET_LEASE_TIME',
    'RECLAIM_COMPLETE',
    'LAYOUTGET',
    'GETDEVICEINFO',
    'LAYOUTCOMMIT',
    'LAYOUTRETURN',
    'SECINFO_NO_NAME',
    'TEST_STATEID',
    'FREE_STATEID',
    'GETDEVICELIST',
    'BIND_CONN_TO_SESSION',
    'DESTROY_CLIENTID',
    'SEEK',
    'ALLOCATE',
    'DEALLOCATE',
    'LAYOUTSTATS',
    'CLONE',
    'COPY',
    'OFFLOAD_CANCEL',
    'LOOKUPP',
    'LAYOUTERROR',
    'COPY_NOTIFY'
]

class DeviceData:
    """DeviceData objects provide methods for parsing and displaying
    data for a single mount grabbed from /proc/self/mountstats
    """
    def __init__(self):
        self.__nfs_data = dict()
        self.__rpc_data = dict()
        self.__rpc_data['ops'] = []

    def __parse_nfs_line(self, words):
        if words[0] == 'device':
            self.__nfs_data['export'] = words[1]
            self.__nfs_data['mountpoint'] = words[4]
            self.__nfs_data['fstype'] = words[7]
            if words[7].find('nfs') != -1 and words[7] != 'nfsd':
                self.__nfs_data['statvers'] = words[8]
        elif 'nfs' in words or 'nfs4' in words:
            self.__nfs_data['export'] = words[0]
            self.__nfs_data['mountpoint'] = words[3]
            self.__nfs_data['fstype'] = words[6]
            if words[6].find('nfs') != -1 and words[6] != 'nfsd':
                self.__nfs_data['statvers'] = words[7]
        elif words[0] == 'age:':
            self.__nfs_data['age'] = int(words[1])
        elif words[0] == 'opts:':
            self.__nfs_data['mountoptions'] = ''.join(words[1:]).split(',')
        elif words[0] == 'caps:':
            self.__nfs_data['servercapabilities'] = ''.join(words[1:]).split(',')
        elif words[0] == 'nfsv4:':
            self.__nfs_data['nfsv4flags'] = ''.join(words[1:]).split(',')
        elif words[0] == 'sec:':
            keys = ''.join(words[1:]).split(',')
            self.__nfs_data['flavor'] = int(keys[0].split('=')[1])
            self.__nfs_data['pseudoflavor'] = 0
            if self.__nfs_data['flavor'] == 6:
                self.__nfs_data['pseudoflavor'] = int(keys[1].split('=')[1])
        elif words[0] == 'events:':
            i = 1
            for key in NfsEventCounters:
                try:
                    self.__nfs_data[key] = int(words[i])
                except IndexError as err:
                    self.__nfs_data[key] = 0
                i += 1
        elif words[0] == 'bytes:':
            i = 1
            for key in NfsByteCounters:
                self.__nfs_data[key] = int(words[i])
                i += 1

    def __parse_rpc_line(self, words):
        if words[0] == 'RPC':
            self.__rpc_data['statsvers'] = float(words[3])
            self.__rpc_data['programversion'] = words[5]
        elif words[0] == 'xprt:':
            self.__rpc_data['protocol'] = words[1]
            if words[1] == 'udp':
                i = 2
                for key in XprtUdpCounters:
                    if i < len(words):
                        self.__rpc_data[key] = int(words[i])
                    i += 1
            elif words[1] == 'tcp':
                i = 2
                for key in XprtTcpCounters:
                    if i < len(words):
                        self.__rpc_data[key] = int(words[i])
                    i += 1
            elif words[1] == 'rdma':
                i = 2
                for key in XprtRdmaCounters:
                    if i < len(words):
                        self.__rpc_data[key] = int(words[i])
                    i += 1
        elif words[0] == 'per-op':
            self.__rpc_data['per-op'] = words
        else:
            op = words[0][:-1]
            self.__rpc_data['ops'] += [op]
            self.__rpc_data[op] = [int(word) for word in words[1:]]
            if len(self.__rpc_data[op]) < 9:
                self.__rpc_data[op] += [0]

    def parse_stats(self, lines):
        """Turn a list of lines from a mount stat file into a 
        dictionary full of stats, keyed by name
        """
        found = False
        for line in lines:
            words = line.split()
            if len(words) == 0:
                continue
            if (not found and words[0] != 'RPC'):
                self.__parse_nfs_line(words)
                continue

            found = True
            self.__parse_rpc_line(words)

    def is_nfs_mountpoint(self):
        """Return True if this is an NFS or NFSv4 mountpoint,
        otherwise return False
        """
        if self.__nfs_data['fstype'] == 'nfs':
            return True
        elif self.__nfs_data['fstype'] == 'nfs4':
            return True
        return False

    def nfs_version(self):
        if self.is_nfs_mountpoint():
            prog, vers = self.__rpc_data['programversion'].split('/')
            return int(vers)

    def display_raw_stats(self):
        """Prints out stats in the same format as /proc/self/mountstats
        """
        print('device %s mounted on %s with fstype %s %s' % \
            (self.__nfs_data['export'], self.__nfs_data['mountpoint'], \
            self.__nfs_data['fstype'], self.__nfs_data['statvers']))
        print('\topts:\t%s' % ','.join(self.__nfs_data['mountoptions']))
        print('\tage:\t%d' % self.__nfs_data['age'])
        print('\tcaps:\t%s' % ','.join(self.__nfs_data['servercapabilities']))
        print('\tsec:\tflavor=%d,pseudoflavor=%d' % (self.__nfs_data['flavor'], \
            self.__nfs_data['pseudoflavor']))
        print('\tevents:\t%s' % " ".join([str(self.__nfs_data[key]) for key in NfsEventCounters]))
        print('\tbytes:\t%s' % " ".join([str(self.__nfs_data[key]) for key in NfsByteCounters]))
        print('\tRPC iostats version: %1.1f p/v: %s (nfs)' % (self.__rpc_data['statsvers'], \
            self.__rpc_data['programversion']))
        if self.__rpc_data['protocol'] == 'udp':
            print('\txprt:\tudp %s' % " ".join([str(self.__rpc_data[key]) for key in XprtUdpCounters]))
        elif self.__rpc_data['protocol'] == 'tcp':
            print('\txprt:\ttcp %s' % " ".join([str(self.__rpc_data[key]) for key in XprtTcpCounters]))
        elif self.__rpc_data['protocol'] == 'rdma':
            print('\txprt:\trdma %s' % " ".join([str(self.__rpc_data[key]) for key in XprtRdmaCounters]))
        else:
            raise Exception('Unknown RPC transport protocol %s' % self.__rpc_data['protocol'])
        print('\tper-op statistics')
        prog, vers = self.__rpc_data['programversion'].split('/')
        if vers == '3':
            for op in Nfsv3ops:
                print('\t%12s: %s' % (op, " ".join(str(x) for x in self.__rpc_data[op])))
        elif vers == '4':
            for op in Nfsv4ops:
                try:
                    print('\t%12s: %s' % (op, " ".join(str(x) for x in self.__rpc_data[op])))
                except KeyError:
                    continue
        else:
            print('\tnot implemented for version %d' % vers)
        print()

    def display_stats_header(self):
        print('Stats for %s mounted on %s:' % \
            (self.__nfs_data['export'], self.__nfs_data['mountpoint']))
        print()

    def display_nfs_options(self):
        """Pretty-print the NFS options
        """
        print('  NFS mount options: %s' % ','.join(self.__nfs_data['mountoptions']))
        print('  NFS mount age: %s' % datetime.timedelta(seconds = self.__nfs_data['age']))
        print('  NFS server capabilities: %s' % ','.join(self.__nfs_data['servercapabilities']))
        if 'nfsv4flags' in self.__nfs_data:
            print('  NFSv4 capability flags: %s' % ','.join(self.__nfs_data['nfsv4flags']))
        if 'pseudoflavor' in self.__nfs_data:
            print('  NFS security flavor: %d  pseudoflavor: %d' % \
                (self.__nfs_data['flavor'], self.__nfs_data['pseudoflavor']))
        else:
            print('  NFS security flavor: %d' % self.__nfs_data['flavor'])

    def display_nfs_events(self):
        """Pretty-print the NFS event counters
        """
        print()
        print('Cache events:')
        print('  data cache invalidated %d times' % self.__nfs_data['datainvalidates'])
        print('  attribute cache invalidated %d times' % self.__nfs_data['attrinvalidates'])
        print()
        print('VFS calls:')
        print('  VFS requested %d inode revalidations' % self.__nfs_data['inoderevalidates'])
        print('  VFS requested %d dentry revalidations' % self.__nfs_data['dentryrevalidates'])
        print()
        print('  VFS called nfs_readdir() %d times' % self.__nfs_data['vfsreaddir'])
        print('  VFS called nfs_lookup() %d times' % self.__nfs_data['vfslookup'])
        print('  VFS called nfs_permission() %d times' % self.__nfs_data['vfspermission'])
        print('  VFS called nfs_file_open() %d times' % self.__nfs_data['vfsopen'])
        print('  VFS called nfs_file_flush() %d times' % self.__nfs_data['vfsflush'])
        print('  VFS called nfs_lock() %d times' % self.__nfs_data['vfslock'])
        print('  VFS called nfs_fsync() %d times' % self.__nfs_data['vfsfsync'])
        print('  VFS called nfs_file_release() %d times' % self.__nfs_data['vfsrelease'])
        print()
        print('VM calls:')
        print('  VFS called nfs_readpage() %d times' % self.__nfs_data['vfsreadpage'])
        print('  VFS called nfs_readpages() %d times' % self.__nfs_data['vfsreadpages'])
        print('  VFS called nfs_writepage() %d times' % self.__nfs_data['vfswritepage'])
        print('  VFS called nfs_writepages() %d times' % self.__nfs_data['vfswritepages'])
        print()
        print('Generic NFS counters:')
        print('  File size changing operations:')
        print('    truncating SETATTRs: %d  extending WRITEs: %d' % \
            (self.__nfs_data['setattrtrunc'], self.__nfs_data['extendwrite']))
        print('  %d silly renames' % self.__nfs_data['sillyrenames'])
        print('  short reads: %d  short writes: %d' % \
            (self.__nfs_data['shortreads'], self.__nfs_data['shortwrites']))
        print('  NFSERR_DELAYs from server: %d' % self.__nfs_data['delay'])
        print('  pNFS READs: %d' % self.__nfs_data['pnfsreads'])
        print('  pNFS WRITEs: %d' % self.__nfs_data['pnfswrites'])

    def display_nfs_bytes(self):
        """Pretty-print the NFS event counters
        """
        print()
        print('NFS byte counts:')
        print('  applications read %d bytes via read(2)' % self.__nfs_data['normalreadbytes'])
        print('  applications wrote %d bytes via write(2)' % self.__nfs_data['normalwritebytes'])
        print('  applications read %d bytes via O_DIRECT read(2)' % self.__nfs_data['directreadbytes'])
        print('  applications wrote %d bytes via O_DIRECT write(2)' % self.__nfs_data['directwritebytes'])
        print('  client read %d bytes via NFS READ' % self.__nfs_data['serverreadbytes'])
        print('  client wrote %d bytes via NFS WRITE' % self.__nfs_data['serverwritebytes'])

    def display_rpc_generic_stats(self):
        """Pretty-print the generic RPC stats
        """
        sends = self.__rpc_data['rpcsends']

        print('RPC statistics:')

        print('  %d RPC requests sent, %d RPC replies received (%d XIDs not found)' % \
            (sends, self.__rpc_data['rpcreceives'], self.__rpc_data['badxids']))
        if sends != 0:
            print('  average backlog queue length: %d' % \
                (float(self.__rpc_data['backlogutil']) / sends))

    def display_rpc_op_stats(self):
        """Pretty-print the per-op stats
        """
        sends = self.__rpc_data['rpcsends']

        allstats = []
        for op in self.__rpc_data['ops']:
            allstats.append([op] + self.__rpc_data[op])

        print()
        for stats in sorted(allstats, key=itemgetter(1), reverse=True):
            count = stats[1]
            if count != 0:
                print('%s:' % stats[0])
                ops_pcnt = 0
                if sends != 0:
                    ops_pcnt = (count * 100) / sends
                print('\t%d ops (%d%%)' % \
                    (count, ops_pcnt), end=' ')
                retrans = stats[2] - count
                if retrans != 0:
                    print('\t%d retrans (%d%%)' % (retrans, ((retrans * 100) / count)), end=' ')
                    print('\t%d major timeouts' % stats[3], end='')
                if len(stats) >= 10 and stats[9] != 0:
                    print('\t%d errors (%d%%)' % (stats[9], ((stats[9] * 100) / count)))
                else:
                    print('')
                print('\tavg bytes sent per op: %d\tavg bytes received per op: %d' % \
                    (stats[4] / count, stats[5] / count))
                print('\tbacklog wait: %f' % (float(stats[6]) / count), end=' ')
                print('\tRTT: %f' % (float(stats[7]) / count), end=' ')
                print('\ttotal execute time: %f (milliseconds)' % \
                    (float(stats[8]) / count))

    def client_rpc_stats(self):
        """Tally high-level rpc stats for the nfsstat command
        """
        sends = 0
        trans = 0
        authrefrsh = 0
        for op in self.__rpc_data['ops']:
            sends += self.__rpc_data[op][0]
            trans += self.__rpc_data[op][1]
        retrans = trans - sends
        # authrefresh stats don't actually get captured in
        # /proc/self/mountstats, so we fudge it here
        authrefrsh = sends
        return (sends, retrans, authrefrsh)

    def display_nfsstat_stats(self):
        """Pretty-print nfsstat-style stats
        """
        sends = 0
        for op in self.__rpc_data['ops']:
            sends += self.__rpc_data[op][0]
        if sends == 0:
            return
        print()
        vers = self.nfs_version()
        print('Client nfs v%d' % vers)
        info = []
        for op in self.__rpc_data['ops']:
            print('%-13s' % str.lower(op)[:12], end='')
            count = self.__rpc_data[op][0]
            pct = (count * 100) / sends
            info.append((count, pct))
            if (self.__rpc_data['ops'].index(op) + 1) % 6 == 0:
                print()
                for (count, pct) in info:
                    print('%-8u%3u%% ' % (count, pct), end='')
                print()
                info = []
        print()
        if len(info) > 0:
            for (count, pct) in info:
                print('%-8u%3u%% ' % (count, pct), end='')
            print()

    def compare_iostats(self, old_stats):
        """Return the difference between two sets of stats
        """
        if old_stats.__nfs_data['age'] > self.__nfs_data['age']:
            return self

        result = DeviceData()
        protocol = self.__rpc_data['protocol']

        # copy self into result
        for key, value in self.__nfs_data.items():
            result.__nfs_data[key] = value
        for key, value in self.__rpc_data.items():
            result.__rpc_data[key] = value

        # compute the difference of each item in the list
        # note the copy loop above does not copy the lists, just
        # the reference to them.  so we build new lists here
        # for the result object.
        for op in result.__rpc_data['ops']:
            try:
                result.__rpc_data[op] = list(map(difference, self.__rpc_data[op], old_stats.__rpc_data[op]))
            except KeyError:
                continue

        # update the remaining keys
        if protocol == 'udp':
            for key in XprtUdpCounters:
                result.__rpc_data[key] -= old_stats.__rpc_data[key]
        elif protocol == 'tcp':
            for key in XprtTcpCounters:
                result.__rpc_data[key] -= old_stats.__rpc_data[key]
        elif protocol == 'rdma':
            for key in XprtRdmaCounters:
                result.__rpc_data[key] -= old_stats.__rpc_data[key]
        result.__nfs_data['age'] -= old_stats.__nfs_data['age']
        for key in NfsEventCounters:
            result.__nfs_data[key] -= old_stats.__nfs_data[key]
        for key in NfsByteCounters:
            result.__nfs_data[key] -= old_stats.__nfs_data[key]
        return result

    def setup_accumulator(self, ops):
        """Initialize a DeviceData instance to tally stats for all mountpoints
        with the same major version. This is for the nfsstat command.
        """
        if ops == Nfsv3ops:
            self.__rpc_data['programversion'] = '100003/3'
            self.__nfs_data['fstype'] = 'nfs'
        elif ops == Nfsv4ops:
            self.__rpc_data['programversion'] = '100003/4'
            self.__nfs_data['fstype'] = 'nfs4'
        self.__rpc_data['ops'] = ops
        for op in ops:
            self.__rpc_data[op] = [0 for i in range(9)]

    def accumulate_iostats(self, new_stats):
        """Accumulate counters from all RPC op buckets in new_stats.  This is
        for the nfsstat command.
        """
        for op in new_stats.__rpc_data['ops']:
            try:
                self.__rpc_data[op] = list(map(add, self.__rpc_data[op], new_stats.__rpc_data[op]))
            except KeyError:
                continue

    def __print_rpc_op_stats(self, op, sample_time):
        """Print generic stats for one RPC op
        """
        if op not in self.__rpc_data:
            return

        rpc_stats = self.__rpc_data[op]
        ops = float(rpc_stats[0])
        retrans = float(rpc_stats[1] - rpc_stats[0])
        kilobytes = float(rpc_stats[3] + rpc_stats[4]) / 1024
        queued_for = float(rpc_stats[5])
        rtt = float(rpc_stats[6])
        exe = float(rpc_stats[7])
        if len(rpc_stats) >= 9:
            errs = int(rpc_stats[8])

        # prevent floating point exceptions
        if ops != 0:
            kb_per_op = kilobytes / ops
            retrans_percent = (retrans * 100) / ops
            rtt_per_op = rtt / ops
            exe_per_op = exe / ops
            queued_for_per_op = queued_for / ops
            if len(rpc_stats) >= 9:
                errs_percent = (errs * 100) / ops
        else:
            kb_per_op = 0.0
            retrans_percent = 0.0
            rtt_per_op = 0.0
            exe_per_op = 0.0
            queued_for_per_op = 0.0
            errs_percent = 0.0

        op += ':'
        print(format(op.lower(), '<16s'), end='')
        print(format('ops/s', '>8s'), end='')
        print(format('kB/s', '>16s'), end='')
        print(format('kB/op', '>16s'), end='')
        print(format('retrans', '>16s'), end='')
        print(format('avg RTT (ms)', '>16s'), end='')
        print(format('avg exe (ms)', '>16s'), end='')
        print(format('avg queue (ms)', '>16s'), end='')
        if len(rpc_stats) >= 9:
            print(format('errors', '>16s'), end='')
        print()

        print(format((ops / sample_time), '>24.3f'), end='')
        print(format((kilobytes / sample_time), '>16.3f'), end='')
        print(format(kb_per_op, '>16.3f'), end='')
        retransmits = '{0:>10.0f} ({1:>3.1f}%)'.format(retrans, retrans_percent).strip()
        print(format(retransmits, '>16'), end='')
        print(format(rtt_per_op, '>16.3f'), end='')
        print(format(exe_per_op, '>16.3f'), end='')
        print(format(queued_for_per_op, '>16.3f'), end='')
        if len(rpc_stats) >= 9:
            errors = '{0:>10.0f} ({1:>3.1f}%)'.format(errs, errs_percent).strip()
            print(format(errors, '>16'), end='')
        print()

    def display_iostats(self, sample_time):
        """Display NFS and RPC stats in an iostat-like way
        """
        sends = float(self.__rpc_data['rpcsends'])
        if sample_time == 0:
            sample_time = float(self.__nfs_data['age'])
        #  sample_time could still be zero if the export was just mounted.
        #  Set it to 1 to avoid divide by zero errors in this case since we'll
        #  likely still have relevant mount statistics to show.
        #
        if sample_time == 0:
            sample_time = 1;
        if sends != 0:
            backlog = (float(self.__rpc_data['backlogutil']) / sends) / sample_time
        else:
            backlog = 0.0

        print()
        print('%s mounted on %s:' % \
            (self.__nfs_data['export'], self.__nfs_data['mountpoint']))
        print()

        print(format('ops/s', '>16') + format('rpc bklog', '>16'))
        print(format((sends / sample_time), '>16.3f'), end='')
        print(format(backlog, '>16.3f'))
        print()

        self.__print_rpc_op_stats('READ', sample_time)
        self.__print_rpc_op_stats('WRITE', sample_time)
        sys.stdout.flush()

    def display_xprt_stats(self):
        """Pretty-print the xprt statistics
        """
        if self.__rpc_data['protocol'] == 'udp':
            print('\tTransport protocol: udp')
            print('\tSource port: %d' % self.__rpc_data['port'])
            print('\tBind count: %d' % self.__rpc_data['bind_count'])
            print('\tRPC requests: %d' % self.__rpc_data['rpcsends'])
            print('\tRPC replies: %d' % self.__rpc_data['rpcreceives'])
            print('\tXIDs not found: %d' % self.__rpc_data['badxids'])
            print('\tMax slots: %d' % self.__rpc_data['maxslots'])
            if self.__rpc_data['rpcsends'] != 0:
                print('\tAvg backlog length: %d' % \
                    (float(self.__rpc_data['backlogutil']) / self.__rpc_data['rpcsends']))
                print('\tAvg send queue length: %d' % \
                    (float(self.__rpc_data['sendutil']) / self.__rpc_data['rpcsends']))
                print('\tAvg pending queue length: %d' % \
                    (float(self.__rpc_data['pendutil']) / self.__rpc_data['rpcsends']))
        elif self.__rpc_data['protocol'] == 'tcp':
            print('\tTransport protocol: tcp')
            print('\tSource port: %d' % self.__rpc_data['port'])
            print('\tBind count: %d' % self.__rpc_data['bind_count'])
            print('\tConnect count: %d' % self.__rpc_data['connect_count'])
            print('\tConnect time: %d seconds' % self.__rpc_data['connect_time'])
            print('\tIdle time: %d seconds' % self.__rpc_data['idle_time'])
            print('\tRPC requests: %d' % self.__rpc_data['rpcsends'])
            print('\tRPC replies: %d' % self.__rpc_data['rpcreceives'])
            print('\tXIDs not found: %d' % self.__rpc_data['badxids'])
            print('\tMax slots: %d' % self.__rpc_data['maxslots'])
            if self.__rpc_data['rpcsends'] != 0:
                print('\tAvg backlog length: %d' % \
                    (float(self.__rpc_data['backlogutil']) / self.__rpc_data['rpcsends']))
                print('\tAvg send queue length: %d' % \
                    (float(self.__rpc_data['sendutil']) / self.__rpc_data['rpcsends']))
                print('\tAvg pending queue length: %d' % \
                    (float(self.__rpc_data['pendutil']) / self.__rpc_data['rpcsends']))
        elif self.__rpc_data['protocol'] == 'rdma':
            print('\tTransport protocol: rdma')
            print('\tConnect count: %d' % self.__rpc_data['connect_count'])
            print('\tConnect time: %d seconds' % self.__rpc_data['connect_time'])
            print('\tIdle time: %d seconds' % self.__rpc_data['idle_time'])
            sends = self.__rpc_data['rpcsends']
            print('\tRPC requests: %d' % self.__rpc_data['rpcsends'])
            print('\tRPC replies: %d' % self.__rpc_data['rpcreceives'])
            print('\tXIDs not found: %d' % self.__rpc_data['badxids'])
            if self.__rpc_data['rpcsends'] != 0:
                print('\tAvg backlog length: %d' % \
                    (float(self.__rpc_data['backlogutil']) / self.__rpc_data['rpcsends']))
            print('\tRead segments: %d' % self.__rpc_data['read_segments'])
            print('\tWrite segments: %d' % self.__rpc_data['write_segments'])
            print('\tReply segments: %d' % self.__rpc_data['reply_segments'])
            print('\tRegistered: %d bytes' % self.__rpc_data['total_rdma_req'])
            print('\tRDMA received: %d bytes' % self.__rpc_data['total_rdma_rep'])
            print('\tTotal pull-up: %d bytes' % self.__rpc_data['pullup'])
            print('\tTotal fix-up: %d bytes' % self.__rpc_data['fixup'])
            print('\tHardway allocations: %d bytes' % self.__rpc_data['hardway'])
            print('\tFailed marshals: %d' % self.__rpc_data['failed_marshal'])
            print('\tBad replies: %d' % self.__rpc_data['bad_reply'])

            """ Counters not present in all kernels """
            if 'nomsg_calls' in self.__rpc_data:
                print('\tRDMA_NOMSG calls: %d' % self.__rpc_data['nomsg_calls'])
            if 'allocated_mrs' in self.__rpc_data:
                print('\tAllocated MRs: %d' % self.__rpc_data['allocated_mrs'])
            if 'recovered_mrs' in self.__rpc_data:
                print('\tRecovered MRs: %d' % self.__rpc_data['recovered_mrs'])
            if 'orphaned_mrs' in self.__rpc_data:
                print('\tOrphaned MRs: %d' % self.__rpc_data['orphaned_mrs'])
            if 'local_invalidates' in self.__rpc_data:
                print('\tLocal Invalidates needed: %d' % self.__rpc_data['local_invalidates'])
            if 'empty_sendctx_q' in self.__rpc_data:
                print('\tEmpty sendctx queue count: %d' % self.__rpc_data['empty_sendctx_q'])
            if 'reply_waits_for_send' in self.__rpc_data:
                print('\tReplies that waited for Send completion: %d' % self.__rpc_data['reply_waits_for_send'])
        else:
            raise Exception('Unknown RPC transport protocol %s' % self.__rpc_data['protocol'])

def parse_stats_file(f):
    """pop the contents of a mountstats file into a dictionary,
    keyed by mount point.  each value object is a list of the
    lines in the mountstats file corresponding to the mount
    point named in the key.
    """
    ms_dict = dict()
    key = ''

    f.seek(0)
    for line in f.readlines():
        words = line.split()
        if len(words) == 0:
            continue
        if words[0] == 'device':
            key = words[4]
            new = [ line.strip() ]
        elif 'nfs' in words or 'nfs4' in words:
            key = words[3]
            new = [ line.strip() ]
        else:
            new += [ line.strip() ]
        ms_dict[key] = new

    return ms_dict

def print_mountstats(stats, nfs_only, rpc_only, raw, xprt_only):
    if nfs_only:
       stats.display_stats_header()
       stats.display_nfs_options()
       stats.display_nfs_events()
       stats.display_nfs_bytes()
    elif rpc_only:
       stats.display_stats_header()
       stats.display_rpc_generic_stats()
       stats.display_rpc_op_stats()
    elif raw:
       stats.display_raw_stats()
    elif xprt_only:
       stats.display_stats_header()
       stats.display_xprt_stats()
    else:
       stats.display_stats_header()
       stats.display_nfs_options()
       stats.display_nfs_bytes()
       stats.display_rpc_generic_stats()
       stats.display_rpc_op_stats()
    print()

def mountstats_command(args):
    """Mountstats command
    """
    mountstats = parse_stats_file(args.infile)
    mountpoints = [os.path.normpath(mp) for mp in args.mountpoints]

    # make certain devices contains only NFS mount points
    if len(mountpoints) > 0:
        check = []
        for device in mountpoints:
            stats = DeviceData()
            try:
                stats.parse_stats(mountstats[device])
                if stats.is_nfs_mountpoint():
                    check += [device]
            except KeyError:
                continue
        mountpoints = check
    else:
        for device, descr in mountstats.items():
            stats = DeviceData()
            stats.parse_stats(descr)
            if stats.is_nfs_mountpoint():
                mountpoints += [device]
    if len(mountpoints) == 0:
        print('No NFS mount points were found')
        return 1

    if args.since:
        old_mountstats = parse_stats_file(args.since)

    for mp in mountpoints:
        stats = DeviceData()
        stats.parse_stats(mountstats[mp])
        if not args.since:
            print_mountstats(stats, args.nfs_only, args.rpc_only, args.raw, args.xprt_only)
        elif args.since and mp not in old_mountstats:
            print_mountstats(stats, args.nfs_only, args.rpc_only, args.raw, args.xprt_only)
        else:
            old_stats = DeviceData()
            old_stats.parse_stats(old_mountstats[mp])
            diff_stats = stats.compare_iostats(old_stats)
            print_mountstats(diff_stats, args.nfs_only, args.rpc_only, args.raw, args.xprt_only)

    args.infile.close()
    if args.since:
        args.since.close()
    return 0

def nfsstat_command(args):
    """nfsstat-like command for NFS mount points
    """
    mountstats = parse_stats_file(args.infile)
    mountpoints = [os.path.normpath(mp) for mp in args.mountpoints]
    v3stats = DeviceData()
    v3stats.setup_accumulator(Nfsv3ops)
    v4stats = DeviceData()
    v4stats.setup_accumulator(Nfsv4ops)

    # ensure stats get printed if neither v3 nor v4 was specified
    if args.show_v3 or args.show_v4:
        show_both = False
    else:
        show_both = True

    # make certain devices contains only NFS mount points
    if len(mountpoints) > 0:
        check = []
        for device in mountpoints:
            stats = DeviceData()
            try:
                stats.parse_stats(mountstats[device])
                if stats.is_nfs_mountpoint():
                    check += [device]
            except KeyError:
                continue
        mountpoints = check
    else:
        for device, descr in mountstats.items():
            stats = DeviceData()
            stats.parse_stats(descr)
            if stats.is_nfs_mountpoint():
                mountpoints += [device]
    if len(mountpoints) == 0:
        print('No NFS mount points were found')
        return 1

    if args.since:
        old_mountstats = parse_stats_file(args.since)

    for mp in mountpoints:
        stats = DeviceData()
        stats.parse_stats(mountstats[mp])
        vers = stats.nfs_version()

        if not args.since:
            acc_stats = stats
        elif args.since and mp not in old_mountstats:
            acc_stats = stats
        else:
            old_stats = DeviceData()
            old_stats.parse_stats(old_mountstats[mp])
            acc_stats = stats.compare_iostats(old_stats)

        if vers == 3 and (show_both or args.show_v3):
           v3stats.accumulate_iostats(acc_stats)
        elif vers == 4 and (show_both or args.show_v4):
           v4stats.accumulate_iostats(acc_stats)

    sends, retrans, authrefrsh = map(add, v3stats.client_rpc_stats(), v4stats.client_rpc_stats())
    print('Client rpc stats:')
    print('calls      retrans    authrefrsh')
    print('%-11u%-11u%-11u' % (sends, retrans, authrefrsh))

    if show_both or args.show_v3:
        v3stats.display_nfsstat_stats()
    if show_both or args.show_v4:
        v4stats.display_nfsstat_stats()

    args.infile.close()
    if args.since:
        args.since.close()
    return 0

def print_iostat_summary(old, new, devices, time):
    for device in devices:
        stats = DeviceData()
        stats.parse_stats(new[device])
        if not old or device not in old:
            stats.display_iostats(time)
        else:
            if ("fstype autofs" not in str(old[device])) and ("fstype autofs" not in str(new[device])):
                old_stats = DeviceData()
                old_stats.parse_stats(old[device])
                diff_stats = stats.compare_iostats(old_stats)
                diff_stats.display_iostats(time)

def iostat_command(args):
    """iostat-like command for NFS mount points
    """
    mountstats = parse_stats_file(args.infile)
    devices = [os.path.normpath(mp) for mp in args.mountpoints]

    if args.since:
        old_mountstats = parse_stats_file(args.since)
    else:
        old_mountstats = None

    # make certain devices contains only NFS mount points
    if len(devices) > 0:
        check = []
        for device in devices:
            stats = DeviceData()
            try:
                stats.parse_stats(mountstats[device])
                if stats.is_nfs_mountpoint():
                    check += [device]
            except KeyError:
                continue
        devices = check
    else:
        for device, descr in mountstats.items():
            stats = DeviceData()
            stats.parse_stats(descr)
            if stats.is_nfs_mountpoint():
                devices += [device]
    if len(devices) == 0:
        print('No NFS mount points were found')
        return 1

    sample_time = 0

    if args.interval is None:
        print_iostat_summary(old_mountstats, mountstats, devices, sample_time)
        return

    if args.count is not None:
        count = args.count
        while count != 0:
            print_iostat_summary(old_mountstats, mountstats, devices, sample_time)
            old_mountstats = mountstats
            time.sleep(args.interval)
            sample_time = args.interval
            mountstats = parse_stats_file(args.infile)
            count -= 1
    else: 
        while True:
            print_iostat_summary(old_mountstats, mountstats, devices, sample_time)
            old_mountstats = mountstats
            time.sleep(args.interval)
            sample_time = args.interval
            mountstats = parse_stats_file(args.infile)

    args.infile.close()
    if args.since:
        args.since.close()
    return 0

class ICMAction(argparse.Action):
    """Custom action to deal with interval, count, and mountpoints.
    """
    def __call__(self, parser, namespace, values, option_string=None):
        if namespace.mountpoints is None:
            namespace.mountpoints = []
        if values is None:
            return
        elif (type(values) == type([])):
            for value in values:
                self._handle_one(namespace, value)
        else:
            self._handle_one(namespace, values)

    def _handle_one(self, namespace, value):
        try:
            intval = int(value)
            if namespace.infile.name != '/proc/self/mountstats':
                raise argparse.ArgumentError(self, "not allowed with argument -f/--file or -S/--since")
            self._handle_int(namespace, intval)
        except ValueError:
            namespace.mountpoints.append(value)

    def _handle_int(self, namespace, value):
        if namespace.interval is None:
            namespace.interval = value
        elif namespace.count is None:
            namespace.count = value
        else:
            raise argparse.ArgumentError(self, "too many integer arguments")

def main():
    parser = argparse.ArgumentParser(epilog='For specific sub-command help, '
        'run \'mountstats SUB-COMMAND -h|--help\'')
    subparsers = parser.add_subparsers(help='sub-command help')

    common_parser = argparse.ArgumentParser(add_help=False)
    common_parser.add_argument('-v', '--version', action='version',
        version='mountstats ' + Mountstats_version)
    common_parser.add_argument('-f', '--file', default=open('/proc/self/mountstats', 'r'),
        type=argparse.FileType('r'), dest='infile',
        help='Read stats from %(dest)s instead of /proc/self/mountstats')
    common_parser.add_argument('-S', '--since', type=argparse.FileType('r'),
        metavar='SINCEFILE',
        help='Show difference between current stats and those in SINCEFILE')

    mountstats_parser = subparsers.add_parser('mountstats',
        parents=[common_parser],
        help='Display a combination of per-op RPC statistics, NFS event counts, and NFS byte counts. '
            'This is the default sub-command if no sub-command is given.')
    group = mountstats_parser.add_mutually_exclusive_group()
    group.add_argument('-n', '--nfs', action='store_true', dest='nfs_only',
        help='Display only the NFS statistics')
    group.add_argument('-r', '--rpc', action='store_true', dest='rpc_only',
        help='Display only the RPC statistics')
    group.add_argument('-R', '--raw', action='store_true',
        help='Display only the raw statistics')
    group.add_argument('-x', '--xprt', action='store_true', dest='xprt_only',
        help='Display only the xprt statistics')
    # The mountpoints argument cannot be moved into the common_parser because
    # it will screw up the parsing of the iostat arguments (interval and count)
    mountstats_parser.add_argument('mountpoints', nargs='*', metavar='mountpoint',
        help='Display statistics for this mountpoint. More than one may be specified. '
            'If absent, statistics for all NFS mountpoints will be generated.')
    mountstats_parser.set_defaults(func=mountstats_command)

    nfsstat_parser = subparsers.add_parser('nfsstat',
        parents=[common_parser],
        help='Display nfsstat-like statistics.')
    nfsstat_parser.add_argument('-3', action='store_true', dest='show_v3',
        help='Show NFS version 3 statistics')
    nfsstat_parser.add_argument('-4', action='store_true', dest='show_v4',
        help='Show NFS version 4 statistics')
    # The mountpoints argument cannot be moved into the common_parser because
    # it will screw up the parsing of the iostat arguments (interval and count)
    nfsstat_parser.add_argument('mountpoints', nargs='*', metavar='mountpoint',
        help='Display statistics for this mountpoint. More than one may be specified. '
            'If absent, statistics for all NFS mountpoints will be generated.')
    nfsstat_parser.set_defaults(func=nfsstat_command)

    iostat_parser = subparsers.add_parser('iostat',
        parents=[common_parser],
        help='Display iostat-like statistics.')
    iostat_parser.add_argument('interval', nargs='?', action=ICMAction,
        help='Number of seconds between reports. If absent, only one report will '
            'be generated.')
    iostat_parser.add_argument('count', nargs='?', action=ICMAction,
        help='Number of reports generated at <interval> seconds apart. If absent, '
            'reports will be generated continuously.')
    # The mountpoints argument cannot be moved into the common_parser because
    # it will screw up the parsing of the iostat arguments (interval and count)
    iostat_parser.add_argument('mountpoints', nargs='*', action=ICMAction, metavar='mountpoint',
        help='Display statsistics for this mountpoint. More than one may be specified. '
            'If absent, statistics for all NFS mountpoints will be generated.')
    iostat_parser.set_defaults(func=iostat_command)
 
    args = parser.parse_args()
    return args.func(args)

try:
    if __name__ == '__main__':
        # Run the mounstats sub-command if no sub-command (or the help flag)
        # is given.  If the argparse module ever gets support for optional
        # (default) sub-commands, then this can be changed.
        if len(sys.argv) == 1:
            sys.argv.insert(1, 'mountstats')
        elif sys.argv[1] not in ['-h', '--help', 'mountstats', 'iostat', 'nfsstat']:
            sys.argv.insert(1, 'mountstats')
        res = main()
        sys.stdout.close()
        sys.stderr.close()
        sys.exit(res)
except (KeyboardInterrupt, RuntimeError):
    sys.exit(1)
except IOError:
    pass