np_treestats.py

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

            if trace:
                inform(str)



############################################################################

    def rank(self):

        self.rank_volume()
        self.rank_pages()
        self.rank_servers()
        self.agtypes.rank(self.rank_str)

############################################################################

    def rank_servers(self):

        #
        # Rank most delayed servers by delay per request/connection
        #

        def add(a, b):
            return a+b

        def by_len1(a, b):
            return len(b[1]) - len(a[1])

        odels = []
        cdels = []
        ns = 0
        nsdel = 0
        nsdelb = 0
        totconn = 0
        ndconn = 0
        nbdconn = 0
        sloss = 0


        d_dupcsyn = 0
        d_dupfreq = 0
        d_cretrans = 0
        d_not_connected = 0
        d_reqnotseen = 0
        d_repnotseen = 0
        serv_oop = 0
        serv_rtmtp = 0

        fnm = self.basepath + 'Server-Conns-Summary'
        try:
            f = open(fnm, 'w')
        except IOError, s:
            print 'Couldn\'t open summary file', fnm, s
            sys.exit(1)
        write = f.write
        
        #print 'Rank most delayed servers by delay per request'
        sdl = self.sdict.items()
        sdl.sort(by_len1)
        #self.nservers = len(sdl)
        for s, clist in sdl:
            #print intoa_string(s), clist
            ns += 1
            delay = 0
            npkts = 0
            nconn = 0
            nd = 0
            nbd = 0
            dflags = 0
            for id in clist:
                dels = self.cdelays[id][0]
                npkts += dels[1]
                nconn += 1
                if len(dels) == 3:
                    ndconn += 1
                    nd += 1
                    dd = dels[2]
                    flags = dd[-1]
                    dflags |= flags
                    for i in range(1, len(dd), 2):
                        delay += dd[i]
                    if flags & D_DUPCSYN:
                        d_dupcsyn += 1
                    if flags & D_DUPFREQ:
                        d_dupfreq += 1
                    if flags & D_CRETRANS:
                        d_cretrans += 1
                    if flags & D_NOT_CONNECTED:
                        d_not_connected += 1
                    if flags & D_REQNOTSEEN:
                        d_reqnotseen += 1
                    if flags & D_REPNOTSEEN:
                        d_repnotseen += 1
                    if flags & D_SLOSS:
                        serv_oop += dd[-3]
                        serv_rtmtp += dd[-2]
                        if dd[-3] or dd[-2]:
                            sloss += 1
                        str = 'TCPConn #%d Server loss:' % (id)
                        self.write_log(str)
                    if flags & D_BIGDEL:
                        nbdconn += 1
                        nbd += 1
                            
            if delay and dflags & D_BIGDEL:
                nsdel += 1
                if npkts > self.nob_thresh:
                    nsdelb += 1
                    odels.append((delay/npkts, s, npkts, clist))
                    cdels.append((delay/nconn, s, nconn, clist))
                    #print intoa_string(s), delay, nob, delay/nob
            totconn += nconn
            #addr, npkts, nconns, nconns-del, nconns-bigdel, delay, dflags 
            write('%s %d %d %d %d %d %d\n' \
                  % (intoa_string(s), npkts, nconn, nd, nbd, delay, dflags))

        f.close()

        for what, list in (('packet', odels), ('connection', cdels)):
            list.sort()
            list.reverse()

            for i in range(min(N_TO_RANK, len(list))):
                rank = self.rank_str(i)
                s = list[i]
                str = 'Rank Server-by-delay/%s %s %s > %d requests (%d - %dms):' % \
                      (what, rank, intoa_string(s[1]), self.nob_thresh, s[2], s[0])

                self.write_log(str)
                self.obdict[str] = (s[1], s[3])

        s = '%d/%d servers showing delays (%d > %d pkts) %d/%d conns with all delays %d with large delays %d persistent conns' \
            % (nsdel, ns, nsdelb, self.nob_thresh, ndconn, totconn, nbdconn,
               self.cpers)
        print s
        self.write_log(s)

        s = '%d dupcsyn %d dupfreq %d cretrans %d not connected %d no req %d no rep %d server loss    ' % (d_dupcsyn, d_dupfreq, d_cretrans, d_not_connected, d_reqnotseen, d_repnotseen, sloss)
        print s
        self.write_log(s)

        

############################################################################

    def rank_volume(self):

        def by_0(a, b):
            return b[0] - a[0]

        def by_1(a, b):
            return b[1] - a[1]

        def by_2(a, b):
            return b[2] - a[2]

        def by_3(a, b):
            return b[3] - a[3]
        
        def by_5(a, b):
            return b[5] - a[5]
        
        def by_6(a, b):
            return b[6] - a[6]
        
        def by_7(a, b):
            return b[7] - a[7]
        
        def by_8(a, b):
            return b[8] - a[8]
        
        def by_9(a, b):
            return int((b[9] - a[9])*100)
        
        def by_10(a, b):
            return int((b[10] - a[10])*100)

        def ustr(q):
            if type(q) == types.FloatType:
                return '%.2f%%' % (q)
            else:
                return '%dms' % (q)

        #
        # Rank clients/servers by pages/connections/objects/bytes
        #


        p_by_c_d = {}
        p_by_s_d = {}

        ranks = [
            [by_0, 'objects', 0],
            [by_1, 'bytes', 1],
            [by_2, 'conns', 2],
            [by_3, 'pages', 3]
            ]

        ends = [[3, 'Client', p_by_c_d, 0], [4, 'Server', p_by_s_d, 1]]

        for ki, end , d , dotm in ends:
            for p in self.pagelist:
                e = d.setdefault(p[ki], [0, 0, 0, 0, [], 0, 0, 0, 0, 0, 0])
                e[0] += p[6] #accumulate nob
                e[1] += p[5] #do bytes
                e[2] += len(p[8]) #do conns
                e[3] += 1 #count page
                e[4].append(p) #add page
                if dotm:
                    dtm = p[7]
                    e[5] += dtm[1]
                    e[6] += dtm[2]
                    e[7] += dtm[3]
                    e[8] += dtm[4]
                    if dtm[1] > dtm[2]:
                        e[9] += 1
                        if dtm[3] > dtm[4]:
                            e[10] += 1

            cl = d.values()

            for sortfn, what, di in ranks:

                cl.sort(sortfn)

                for i in range(min(N_TO_RANK, len(cl))):
                    rank = self.rank_str(i)
                    stuff = cl[i]
                    adr = stuff[4][0][ki]
                    #print stuff[:4]
                    #print 'rank', end, what, i, cli[0]
                    str = 'Rank %s-by-#%s %s %s (%d):' % (end, what, rank , intoa_string(adr), stuff[di])
                    self.write_log(str)
                    #self.obdict[str] = (adr, stuff[4])
                    self.obdict[str] = stuff[4]


        
        sv = p_by_s_d.values()
        sv.sort(by_3)
        svl = []
        for s in sv:
            nob = s[0]
            npages = s[3]
            if nob/npages > self.nob_thresh:
                svl.append(s)
            #else:
                #continue
            s[9] *= (100.0/npages)
            s[10] *= (100.0/npages)
            s[5] /= nob
            s[6] /= nob
            s[7] /= nob
            s[8] /= nob

        nppstr = '-(>%dOb/P)' % (self.nob_thresh)
        s1 = 'Rank Server-by-pdt-per-object-(>%dOb/P)' % (self.nob_thresh)
        s2 = 'Rank server-by-%%-pages-delayed-(>%dOb/P)' % (self.nob_thresh)

        ranks = [
            [by_5, s1, '', 0, 5],
            [by_6, s1, '-less-delays', 0, 6],
            [by_7, s1, '-85%', 0, 7],
            [by_8, s1, '-85%-less-delays', 0, 8],
            [by_9, s2, '', 3, 9],
            [by_10, s2, '-85%', 3, 10]
            ]

        for sortfn, rs, what, ti, di in ranks:

            svl.sort(sortfn)

            for i in range(min(N_TO_RANK, len(svl))):
                rank = self.rank_str(i)
                stuff = svl[i]
                if stuff[di] == 0:
                    break
                adr = stuff[4][0][4]
                #print stuff[:4]
                #print 'rank', end, what, i, cli[0]
                str = '%s%s %s %s (%d - %s):' % (rs, what, rank , intoa_string(adr), stuff[ti], ustr(stuff[di]))
                self.write_log(str)
                #self.obdict[str] = (adr, stuff[4])
                self.obdict[str] = stuff[4]

        self.save_delayed_servers([s for s in svl if s[9]])
            
            

############################################################################

    def rank_pages(self):

        #
        # Rank pages by frequency, download times per object
        #


        def by_5_len(a, b):
            return len(b[5]) -len(a[5])

        def by_0(a, b):
            return b[0] -a[0]

        def by_1(a, b):
            return b[1] -a[1]

        def by_2(a, b):
            return b[2] -a[2]

        def by_3(a, b):
            return b[3] -a[3]

        def by_6(a, b):
            return b[6] -a[6]

        def by_7(a, b):
            return b[7] -a[7]

        def by_8(a, b):
            return b[8] -a[8]

        d = {}
        for p in self.pagelist:
            e = d.setdefault(p[0], [0, 0, 0, 0, 0, [], 0, 0, 0])
            e[5].append(p) # add page
            tms = p[7]
            dflags = tms[5]
            e[0] += tms[1] #accumulate dur
            e[1] += tms[2] # do less delays
            e[2] += tms[3] # do 85%
            e[3] += tms[4] # do less delays
            e[4] += p[6] # do no. obs
            e[7] |= dflags # delay flags
            if dflags & D_BIGDEL:
                e[8] += 1

        allpl = d.values()
        self.nurl = len(allpl)
        pl = []
        allpl.sort(by_5_len)           

        # by number of times seen
        pl = [p for p in allpl if p[4]/len(p[5]) > self.nob_thresh]
        #pl.sort(by_5_len)

        for i in range(min(N_TO_RANK, len(pl))):
            rank = self.rank_str(i)
            pages = pl[i][5]
            url = pages[0][0]
            str = 'Rank Pages-(>%dob)-by-frequency %s %s (%d):      ' % (self.nob_thresh, rank , url, len(pages))
            self.write_log(str)
            self.obdict[str] = pages

        # by download time per object
        for p in pl:
            p.append(p[0]-p[1])
            p.append(p[2]-p[3])
            nob = p[4]
            p[0] /= nob
            p[1] /= nob
            p[2] /= nob
            p[3] /= nob
            p[6] = nob/len(p[5])
            p[7] /= nob
            p[8] /= nob

        ranks = [
            ['download-time-per-object', by_0, 0, 'ms'],
            ['download-time-per-object-less-delays', by_1, 1, 'ms'],
            ['85%-download-time-per-object', by_2, 2, 'ms'],
            ['85%-download-time-per-object-less-delays', by_3, 3, 'ms'],
            ['average-page-size', by_6, 6, 'objects'],
            ['delay-per-object', by_7, 7, 'ms'],
            ['85%-delay-per-object', by_8, 8, 'ms']
            ]

        
        for what, sfn, indx, dim in ranks:
            pl.sort(sfn)
            for i in range(min(N_TO_RANK, len(pl))):
                if pl[i][indx] == 0:
                    break
                rank = self.rank_str(i)
                pages = pl[i][5]
                url = pages[0][0]
                str = 'Rank Pages-(>%dob)-by-%s %s %s (%d %d %s):' % (self.nob_thresh, what, rank , url,
                                                   len(p[5]), pl[i][indx], dim)
                self.write_log(str)
                self.obdict[str] = pages

############################################################################