np_treestats.py

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

###############################################################################
#                                                                             #
#   Copyright 2005 University of Cambridge Computer Laboratory.               #
#                                                                             #
#   This file is part of Nprobe.                                              #
#                                                                             #
#   Nprobe is free software; you can redistribute it and/or modify            #
#   it under the terms of the GNU General Public License as published by      #
#   the Free Software Foundation; either version 2 of the License, or         #
#   (at your option) any later version.                                       #
#                                                                             #
#   Nprobe is distributed in the hope that it will be useful,                 #
#   but WITHOUT ANY WARRANTY; without even the implied warranty of            #
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the             #
#   GNU General Public License for more details.                              #
#                                                                             #
#   You should have received a copy of the GNU General Public License         #
#   along with Nprobe; if not, write to the Free Software                     #
#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA #
#                                                                             #
###############################################################################


##############################################################################
## 
##
## Stats collector for HTTP reference tree analysis
## 
##
## 
############################################################################
import types
from nprobe import intoa_string, _inet_aton
from np_filerec import FileRec
from np_notesel import Sellist, CallBackError
from np_TCPDisplay import *
from np_TScreen import Tscreen
from np_statscollector import StatsCollector
from np_plot import DataSet, np_Plot, DATA_TS
from ts_smooth import TsSmooth, NoDataError, SmoothError
from nprobe import TCP_SERV_SYN, TCP_CLI_SYN, TCP_SERV_HTTP, SERVER, CLIENT, \
     TRANS_VAL, TRANS_DUMMY_UNSYNCH, TRANS_DUMMY_ERR, TRANS_ERR, \
     TRANS_INCOMPLETE, TRANS_FINISHED
from nprobe import TSP_SYN, TSP_DUP_SYN, TSP_FIN, TSP_ACKSYN, TSP_RST, \
     CT_NON_OBJECT
from np_TCPConn import D_DUPCSYN, D_DUPFREQ, D_CRETRANS, D_NOT_CONNECTED, \
     D_REQNOTSEEN, D_REPNOTSEEN, D_SLOSS, D_BIGDEL
from np_btypes import AgentTypes, SA, UA
from TCP_Imp import TCP_Imps
from np_atypes import Atype

from array import *
from print_col import whoops, inform

############################################################################

#
# Constants
#

N_TO_RANK = 50 # Top N to rank
NOB_THRESH = 5 # Threshold for no. objects to count a page

PPEROB_TOP = 1000

############################################################################


class TreeStats(StatsCollector):

    def __init__(self, args, trace=0, quiet=0, savedata=1, rec_obs=0, logpath=None):

        StatsCollector.__init__(self, args, quiet=quiet, trace=trace,
                                logpath=logpath)

        self.nob_thresh = NOB_THRESH
        self.nlpages = 0
        self.nspages = 0
        self.spdict = {}
        self.pagelist = []
        self.npages = 0
        self.npagesb = 0
        self.npagesd = 0
        self.npagesdb = 0
        self.cdelays = {}
        self.del_only = 0
        self.ssdeld = None
        self.serv_assoc = {}

        self.nclients = 0
        self.nclients_constructed = 0
        self.nservers = 0

        self.ppers = 0
        self.cpers = 0
        self.nob = 0
        self.nobpers = 0
        self.nconns = 0
        self.nurl = 0

        self.uconns = 0
        self.uconns_inc = 0
        self.ob_uninc = 0
        self.c_uninc = 0
        self.userv = 0
        self.uservd = {}

        self.pperob = array('i', [0]*(PPEROB_TOP+2))
        self.pperob_c = array('i', [0]*(PPEROB_TOP+2))

        self.agtypes = AgentTypes(self.write_log, N_TO_RANK)
        self.reg_b = self.agtypes.reg_b

        self.savedata = savedata
        self.robs = rec_obs
        #self.quiet = quiet

        nppstr = '-(>%dOb/P)' % (self.nob_thresh)

        self.writepage = self.writeconn = self.writepperob = self.nullf

        self.tcp_imps = TCP_Imps()
        self.adict = Atype()

        tree_and_conngroups = [
            'Rank Client-by-#objects',
            'Rank Client-by-#bytes',
            'Rank Client-by-#conns',
            'Rank Client-by-#pages',
            
            'Rank Server-by-#objects',
            'Rank Server-by-#bytes',
            'Rank Server-by-#conns',
            'Rank Server-by-#pages',
            
            'Rank Client/Server-by-#objects',
            'Rank Client/Server-by-#bytes',
            'Rank Client/Server-by-#nconns',
            'Rank Server-by-delay/connection',
            'Rank Server-by-delay/request',
            'Rank Server-by-pdt-per-object' + nppstr + '',
            'Rank Server-by-pdt-per-object' + nppstr + '-less-delays',
            'Rank Server-by-pdt-per-object' + nppstr + '-85%',
            'Rank Server-by-pdt-per-object' + nppstr + '-85%-less-delays',
            'Rank server-by-%-pages-delayed' + nppstr + '',
            'Rank server-by-%-pages-delayed' + nppstr + '-85%'
            ]

        #for w in tree_and_conngroups:
            #print w

        conngroup = ['TCPConn'] + tree_and_conngroups

        treegroup = ['WebClient'] + tree_and_conngroups
        
        nppstr = 'Rank Pages-(>%dob)' % (self.nob_thresh)
        pagegroup = [
            nppstr + '-by-frequency',
            nppstr + '-by-download-time-per-object',
            nppstr + '-by-download-time-per-object-less-delays',
            nppstr + '-by-85%-download-time-per-object',
            nppstr + '-by-85%-download-time-per-object-less-delays',
            nppstr + '-by-average-page-size',
            nppstr + '-by-delay-per-object',
            nppstr + '-by-85%-delay-per-object'
            ]

        allpagegroup = pagegroup + treegroup

        ptgroup = [ 
            'All page times', 
            'All page times - pages <= %d' % (self.nob_thresh), 
            'All page times - pages > %d' % (self.nob_thresh)
            ] 
        
        dtmgroup = tree_and_conngroups + ptgroup + pagegroup

        dpgroup = tree_and_conngroups + pagegroup

        rtmtgroup = ['Retransmits'] + dpgroup
        delgroup = ['Delays', 'Delays-ob'] + dpgroup
        attemptsgroup = ['Attempts']  + dpgroup

        #
        # This is the mapping between packet triggers, stats draw (extract)
        # buttons, and button groups
        # fields: [0] type/button text, [1] groups list, [2] trigger,
        #  [3] delimiter after button flag, [4] plot title
        #
        self.extract_what = [
            ('Connections', conngroup,  None, 0, ''),
            #('Trees', treegroup,  None, 1, ''),
            ('Page', allpagegroup, None, 0, ''),
            ('All Pages', allpagegroup, None, 1, ''),

            
            ('Page Times', dtmgroup, None, 0, 'Page download times'),
            ('85% Page Times', dtmgroup, None, 0, '85% Page download times'),
            ('Page Times less delays', dtmgroup, None, 0,
             'Page download times less delays'),
            ('85% Page Times less delays', dtmgroup, None, 1,
             '85% Page download times less delays'),

            
            ('Page Times per-object', dtmgroup, None, 0,
             'Page download times per-object'),
            ('85% Page Times per-object', dtmgroup, None, 0,
             '85% Page download times per-object'),
            ('Page Times less delays per-object', dtmgroup, None, 0,
             'Page download times less delays per-object'),
            ('85% Page Times less delays per-object', dtmgroup, None, 1,
             '85% Page download times less delays per-object'),

            
            ('Retransmits', rtmtgroup, None, 0, 'Retransmits'),
            ('Retransmits-P-over-1s', rtmtgroup, None, 0, 'Retransmits'),
            ('Retransmits-P-over-5s', rtmtgroup, None, 0, 'Retransmits'),
            ('Retransmits-P-over-10s', rtmtgroup, None, 0, 'Retransmits'),
            ('Retransmits-P-over-60s', rtmtgroup, None, 1, 'Retransmits'),
            ('Delays', delgroup, None, 0, 'Delays'),
            ('Page-delays', dtmgroup, None, 0, 'Delays per page'),
            ('Page-delays-per-object', dtmgroup, None, 0, 'Delays per object per page'),
            ('Attempts', attemptsgroup, 1, 'Attempts')
            ]

############################################################################


    def conn_dels(self, cli):

        #
        # find delays associated with each connection
        #

        def add_ppo(ppo,np):

            try:
                ppo[np] += 1
            except IndexError:
                ppo[-1] += 1
               
            

        self.nconns += len(cli.connlist)
        uconns = []
        dflags = 0
        pperob = self.pperob
        pperob_c = self.pperob_c
        for c in cli.connlist:
            c.pageno = 0
            gotreq, df = c.calc_delays(cli.addr, self.tcp_imps, self.adict)
            dflags |= df
            if not gotreq or not c.translist:
                uconns.append(c)
                continue


            if c.nsgood > 1 or c.ncgood > 1:
                c.persist = max(c.nsgood, c.ncgood)
                self.cpers += 1
                self.nobpers += c.persist
                contlist = []
                i = 0
                for t in c.translist:
                    if t.iscontainer:
                        contlist.append(i)
                    i += 1
                # this is a quick hack - no good for pipelined conns
                first = 1
                to = 0
                opkt = 0
                for p in c.pktlist:
                    if not p.len:
                        continue
                    way = p.dir
                    if way == CLIENT:
                        if first:
                            first = 0
                        else:
                            add_ppo(pperob, opkt)
                            if to in contlist:
                                add_ppo(pperob_c, opkt)
                                
                        opkt = 0
                        to += 1
                    else:
                        opkt += 1
                # collect last
                add_ppo(pperob, opkt)
                if to in contlist:
                    add_ppo(pperob_c, opkt)
                        
            else:
                add_ppo(pperob, c.sdpkts)
                if c.translist[0].iscontainer:
                    add_ppo(pperob_c, c.sdpkts)

        return (uconns, dflags)

############################################################################

    def remdels(self, conns):

        def add(a, b):
            return a+b

        def insert(l, t):
            i = 0
            for lt in l:
                if lt[0] < t[0]:
                    break
                i += 1
            l.insert(i, t)

        trace = 0 and self.trace
        #trace = 1

        if trace:
            print 'new'
        conni = 0
        closes = []
        ncloses = []
        nop = 0
        oop = 0
        tdel = 0

        c = conns[0]
        start = c.open/1000
        nend = 0
        tt = ntt = c.open/1000
        trig = ntrig = 0

        for c in conns:
            id = c.indx
            d =  c.get_delays(1)
            cop = c.cop2
            ot = c.open/1000
            #ct = c.close/1000
            ct = c.close1/1000
            if d and d[0] & D_REPNOTSEEN:
                norep = 1
                if trace:
                    print id, 'no response'
            else:
                norep = 0
            dur = ct-ot
            #print c.id, cop, ot
            while oop > cop:
                gr = closes.pop()
                tt = gr[0]
                trig = gr[1]
                oop -= 1
                #print 'old pop', c.id, tt,
            lag = ot - tt
            #print id, 'trig', trig
            insert(closes, (ot+dur, id))
            oop += 1
            if d and d[0] & D_BIGDEL:
                #print d
                dr = reduce(add, d[1:])
                tdel += dr
                if norep:
                    dur = dr
                dur -= dr
            else:
                dr = 0
            while nop > cop:
                ngr = ncloses.pop()
                ntt = ngr[0]
                ntrig = ngr[1]
                nop -= 1
                #print 'pop', c.id, ot
            newot = ntt + lag
            nc = newot+dur
            if trace:
                print 'dur-dr=', dur+dr, dr, 'newot+dur=nc', newot, dur, nc
            insert(ncloses, (nc, id))
            nop +=1
            c.dsave = ot - newot
            if trace:
                #print '%3d %2d %4d %6d->%6d (%5d+%5d) %6d->%6d %2d->%2d %d %d' % (id, cop, dr,  c.open/1000, newot, ntt, lag,  c.close/1000, nc, trig, ntrig, nc-c.close/1000, nend)
                print '%3d %2d %4d %6d->%6d (%5d+%5d) %6d->%6d %2d->%2d %d' % (id, cop, dr,  c.open/1000, newot, ntt, lag,  c.close/1000, nc, trig, ntrig, nc-c.close/1000)
            
        return tdel    

############################################################################
        

    def suck_tree(self, client, uconns):

        def add(a, b):
            return a+b