np_treestats.py

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

    def write_pperob(self):

        for a, suff in [(self.pperob, ''),
                            (self.pperob_c, '_containers')]:

            fnm = self.basepath + 'Pkts-per_ob' + suff + '.hist'
            try:
                f = open(fnm, 'w')
            except IOError, s:
                print 'Couldn\'t open Pkts-per_ob file', fnm, s
                sys.exit(1)
            write = f.write

            po = a
            for i in range(1, len(po)-1):
                n = po[i]
                if n:
                    write('%d %d\n' % (i, n))
            if po[-1]:
                write('#gt %d %d' % (PPEROB_TOP, po[-1]))

            try:
                f.close()
            except IOError, s:
                print 'Couldn\'t close Pkts-per_ob file', fnm, s
                

############################################################################

    def other_results(self):

        #
        # Enter selections for other results into notes
        #
        
        def pass_lte(p):
            return p[6] <= self.nob_thresh

        def pass_gt(p):
            return p[6] > self.nob_thresh
        
        logfn = self.write_log

        #
        # Entries for page download times (all pages)
        #

        # all pages seen
        str = 'XMsg All Page Downloads: All page times: All page downloads:  All page downloads (%d)' % (len(self.pagelist)) 
        logfn(str)
        self.obdict['All page downloads (%d)' \
                    % (len(self.pagelist))] = None

        # all pages seen <= N objects
        str = 'XMsg All Page Downloads: All page times - pages <= %d: All page downloads - pages <= %d:  All page downloads - pages <= %d (%d)' \
              % (self.nob_thresh, self.nob_thresh, self.nob_thresh, self.nspages) 
        logfn(str)
        self.obdict['All page downloads - pages <= %d (%d)' \
                    % (self.nob_thresh, self.nspages)] = pass_lte

        # all pages seen > N objects
        str = 'XMsg All Page Downloads: All page times - pages > %d: All page downloads - pages > %d:  All page downloads - pages > %d (%d)' \
              % (self.nob_thresh, self.nob_thresh, self.nob_thresh, self.nlpages) 
        logfn(str)
        self.obdict['All page downloads - pages > %d (%d)' \
                    % (self.nob_thresh, self.nlpages)] = pass_gt

        #
        # Entries for Connection delays etc.
        #
        logfn('XMsg Retransmits/connection over time: Retransmits: X: X')
        logfn('XMsg Delays/connection over time: Delays: X: X')
        logfn('XMsg Delays/object over time: Delays-ob: X: X')
        logfn('XMsg No. attempts/connection: Attempts: X: X')

        #
        # Other stuff
        #

        str = '%d Pages seen (%d > %d objects) %d with delays (%d > %d objects) %d persistent  ' % (self.npages, self.npagesb, self.nob_thresh, self.npagesd, self.npagesdb, self.nob_thresh, self.ppers)
        print str
        self.write_log(str)

        
##         #
##         # Save page info
##         #
##         if not self.quiet:
##             fnm = self.basepath + 'NOBT%d.Pages' % (self.nob_thresh)
##             try:
##                 f = open(fnm, 'w')
##             except IOError, s:
##                 print 'Page Summary - couldn\'t open', fnm, s
##                 sys.exit(1)
##             write = f.write

##             for p in self.pagelist:
##                 url = p[0].replace(' ', '\32')
##                 write('%s %s %s %d %d %d ' % (url, intoa_string(p[3]),
##                                       intoa_string(p[4]), p[5], p[6], len(p[8])))
##                 dtm = p[7]
##                 write('%.3f ' % (dtm[0]))
##                 for v in dtm[1:]:
##                     write('%d ' % (v))
##                 write('\n')
##             f.close()

##             fnm = self.basepath + 'Connections'
##             try:
##                 f = open(fnm, 'w')
##             except IOError, s:
##                 print 'open save files - couldn\'t open', fnm, s
##                 sys.exit(1)
##             write = f.write

##             for c in self.cdelays.items():
##                 write('%s ' % (c[0]))
##                 dly = c[1][0]
##                 write('%d %d %d' % (dly[0], dly[1], c[1][1]))
##                 if len(dly) == 3:
##                     for v in dly[2]:
##                         write(' %d' % (v))
##                 write('\n')
                    




############################################################################

    def save_delayed_servers(self, sl):

        sd = self.load_delayed_servers()

        fnm = self.basedir + 'delayed_servers_NOBT%d' % (self.nob_thresh)
        try:
            f = open(fnm, 'w')
        except IOError, s:
            print 'save_delayed_servers - couldn\'t open', fnm, s
            sys.exit(1)
            
        for s in sl:
            pages = s[4]
            first = pages[0]
            d = sd.setdefault(first[4], {})
            for p in pages:
                d[p[3]] = 1
            
        write = f.write
        for s in sd.items():
            write('%s:' % (intoa_string(s[0])))
            for c in s[1].keys():
                write('%s ' % (intoa_string(c)))
            write('\n')

        f.close()

############################################################################

    def load_delayed_servers(self):

        if self.ssdeld != None:
            # already loaded
            return self.ssdeld
        
        self.ssdeld = sd = {}

        fnm = self.basedir + 'NOBT%d.delayed_servers' % (self.nob_thresh)
        try:
            f = open(fnm, 'r')
        except IOError, s:
            print 'get_delayed_servers - couldn\'t open', fnm, s
            if s.__str__().find('No such file or directory') >= 0:
                print '\t- will create'
                return sd
            else:
                sys.exit(1)

        #self.ssdeld = sd = {}
        for l in f.readlines():
            s, cs = l.split(':')
            d = sd[_inet_aton(s)] = {}
            for c in cs.split(' '):
                d[_inet_aton(c)] = 1
        
        f.close()

        return sd

############################################################################

    def get_delayed_servers(self):

        sdict = {}
        cdict = {}
        sd = self.load_delayed_servers()
        for s in sd.items():
            sdict[s[0]] = 1
            for c in s[1]:
                cdict[c] = 1

        return (sdict, cdict)

############################################################################
    

    def save_server_assoc(self):

        fnm = self.basedir + 'server_associations'
        try:
            f = open(fnm, 'w')
        except IOError, s:
            print 'save_server_assoc - couldn\'t open', fnm, s
            sys.exit(1)

        for s in self.serv_assoc.items():
            f.write(intoa_string(s[0]) + ': ')
            for a in s[1].keys():
                if a:
                    f.write(intoa_string(a) + ' ')
            f.write('\n')

        f.close()


############################################################################

    def load_server_assoc(self):

        fnm = self.basedir + 'server_associations'
        try:
            f = open(fnm, 'r')
        except IOError, s:
            #print 'load_server_assoc - couldn\'t open', fnm, s
            print s.__str__()
            if s.__str__().find('No such file or directory') >= 0:
                print '\t- will create'
                return
            else:
                sys.exit(1)

        self.serv_assoc = sa = {}
        for li in f.readlines():
            b, al = li.split(':')
            d = sa[_inet_aton(b)] = {}
            for s in al.split(' '):
                d[_inet_aton(s)] = 1
                         
        f.close()

        return sa

############################################################################

    def add_server_assoc(self, sdict):

        ad = self.serv_assoc

        for sa in ad.keys():
            if sa in sdict:
                for s in ad[sa].keys():
                    sdict[s] = 1

############################################################################

    def add_strict_server_assoc(self, sdict):

        ad = self.serv_assoc

        for sa in ad.keys():
            if sa in sdict:
                b = sdict[sa]
                for add in ad[sa].keys():
                    d = sdict.setdefault(add, {})
                    d.update(b)
            


############################################################################

    #
    # Call-back for Sellist if desired to examine object(s)
    #   - reconstructs from file records and displays

    def draw_objects(self, what, oblist, obdict):

        def recons_type(k):

            for key, type in [ \
            ('Client/Server', 'WebHosts'), \
            ('Client-', 'WebClient'), \
            ('Server-', 'WebServer'), \
            ('Pages-', 'WebClient')
            ]:
                if k.find(key) > 0:
                    return type

            
            print 'Unknown Rank Client filerec type \'%s\'' % (k)
            sys.exit(1)

        def get_conns_from_plist(pl):
            conns = []
            for p in pl:
                conns.extend(p[8])
            return conns
        
        def get_ptlist_from_plist(pl, pfn):

            def pass_all(p):
                return 1

            if not pfn:
                pfn = pass_all
                
            d = {}
            for p in pl:
                if pfn(p):
                    e = d.setdefault(p[3], [])
                    e.append(p)
            ptl = []
            for e in d.items():
                ptl.append(('WebClient', e[1], e[0]))
            return ptl

        def make_pages(plist):
            for p in pages:
                ad = p[3]
                tlist.append(FileRec('WebClient',
                     [obdict[c].recsets for c in p[8]], addr=ad,
                                         rooturl=None, lookup=lookup))

        def make_all_pages(plist):
            for p in plist:
                ad = p[3]
                rooturl = (p[0], p[1])
                conns = self.cdict[ad]
                tlist.append(FileRec('WebClient',
                             [obdict[c].recsets for c in conns], addr=ad,
                             rooturl=rooturl, lookup=lookup))

        lookup = self.lookup
        lookup2 = self.lookup2
        
        tlist = []
        clist = []
        ptlist = []
        ptdiv = 0

        print 'what=', what
        #print oblist

        for m in oblist:
            print 'm=', m
            if m[2] == 'TCPConn':
                clist.append(obdict[m[0]].reconstruct(trace=1))
            elif m[2] == 'WebClient':
                tlist.append(FileRec('WebClient',
                       [obdict[c].recsets for c in self.cdict[m[0]]],
                                     lookup=lookup))
            elif m[2].find('Pages-by-frequency') > 0 \
                     or m[2].find('Pages-by-download-time-per') >= 0 \
                     or m[2].find('Pages-by-85%-download-time-per') >= 0 \
                     or m[2].find('Pages-by-average-page-size') >= 0:
                pages = obdict[m[0]]

                if what == 'Connections':
                    conns = get_conns_from_plist(pages)
                    clist.extend(\
                    [obdict[c].reconstruct(trace=1) for c in conns])
                elif what == 'Page':
                    make_pages(pages)
                elif what == 'All Pages':
                    make_all_pages(pages)
                elif what.find('Page-delays') >= 0:
                    if what.find('per_object') >= 0:
                        po = 1
                    else:
                        po = 0
                    self.show_page_delays(pages, po)
                elif what.find('Page Times') >= 0:
                    if what.find('less delays') >= 0:
                        pti = 2
                        label = 'Page download times less delays'
                    else:
                        pti = 1
                        label = 'Page download times'
                    if what.find('85%') >= 0:
                        pti += 2
                        label = '85% ' + label
                    ptlist.extend(get_ptlist_from_plist(pages, None))

                elif what.find('Retransmits') >= 0 \