np_webhost.py

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

        l = trans.reflink

        if l:
            #print 'fail link from %d type %x' % (l.trans.order, l.type)

            # always insert sub at original fail - follow links back
            while l and l.type == LR_FOLLOW_ON and l.trans.reflink:
                l = l.trans.reflink
                #print 'fail link now from %d type %x' % (l.trans.order, l.type)
            i = 0
            #print l.subs
            if not len(l.subs):
                l.subs.append([trans.repend, LR_FOLLOW_ON, None, trans])
                if trace:
                    print '%d inserting fail at sub0' % (trans.order)
            else:
                # skip over any subs already followed, then insert this
                for s in l.subs:
                    if (not s[2]) and (not s[1] == LR_FOLLOW_ON):
                        #no target = not followed
                        break
                    i += 1
                # insert before first non-followed sub link
                l.subs.insert(i, [trans.repend, LR_FOLLOW_ON, None, trans])
                if trace:
                    print '%d inserting fail at sub%d' % (trans.order, i)

            str = 'WebClient #%s Fail registered: (%d)' % \
                  (self.addr_str(), trans.order)
            self.logfun(str)
            if trace:
                inform(str)

        
            

            # XXX TODO - get this picked up as dependancy for any re-fetch
#############################################################################

    def build_nslist(self):

        #print self.lookups
        #raw_input('...')

        # **idempotent**

        trace = 1 and self.trace

        # connlist must be sorted before calling

	def cmp_lookup_by_reqtm(a, b):
            #print 'cmp_lookup_by_reqtm', a.absreqtm, b.absreqtm
	    return int(a.absreqtm - b.absreqtm)

	if not len(self.lookups):
	    return
        
	tconns = self.connlist
	start = self.start

	#debugging stuff
	sdict = {}
	for conn in tconns:
	    if not sdict.has_key(conn.server):
		sdict[conn.server] = 0
	addrs = sdict.keys()
	addrs.sort()
	#for s in addrs:
	    #print '%s ' % (nprobe.intoa_string(s))

	sdict = {}
	lookupstart = self.lookups[0].absreqtm
	lastm = self.lookups[0].absreptm
	for l in self.lookups:
	    #print l.absreptm
	    if l.absreptm < lastm:
                str = 'WebClient #%s Lookups not ordered: last %d this %d' % \
		      (self.addr_str(), int(lastm - lookupstart), 
		       int(l.absreptm - lookupstart))
                if trace:
                    whoops(str)
                self.logfun(str)
		#sys.exit(1)
	    lastm = l.absreptm
	    for r in l.rrlist:
		if r.code == 1: 
		    if not sdict.has_key(r.addr):
			sdict[r.addr] = 0
	#for s in addrs:
	    #print '%s: ' % (nprobe.intoa_string(s))

	# the real meat
	nslist = self.nslist
	self.lookups.sort(cmp_lookup_by_reqtm)
	for l in self.lookups:
            #print 'NSL'
            #raw_input('...')
	    thisconn = None
	    for r in l.rrlist:
		if r.code == 1:
		    tm = l.absreptm
		    for conn in tconns:
			if r.addr == conn.server and tm < conn.abstart and not conn.ns_attached:
			    thisconn = conn
			    break
	    if thisconn:
		thisconn.lookup = l # not idempotent
		thisconn.ns_attached = 1 # not idempotent
		nslist.append(l)
		#print 'attaching lookup %s to conn #%d %s' % (nprobe.intoa_string(r.addr), thisconn.indx, nprobe.intoa_string(thisconn.server))
		if l.absreqtm < start:
		    start = l.absreqtm

	self.start = start

#############################################################################

#
# Adjust all timings to offsets from when Client first entered our ken,
# biased by value of self.tmbase (e.g. to relate all trees to common start)
#

    def adjust_tm_offsets(self):

        # **idempotent**

	# self.start = absolute first connection open or ns lookup for this
        # client
        
        # bias for common start time if given, else all relative to self.start
        if self.tmbase == None:
            self.tmbase = self.start
            
	self.start -= self.tmbase
        self.end -= self.tmbase
        start = self.tmbase

	for conn in self.connlist:
	    conn.adjust_tm_offsets(start)
         
        #print 'adjust_tm_offsets: translist'  
	for req in self.translist:
	    # relative to open -> relative to start
            req.TConnopen = open = req.abtmbase-start 
	    req.reqstart = open + req.relreqstart
            #print req.reqstart
	    req.reqend = open + req.relreqend
	    repstart = req.repstart = open + req.relrepstart
	    req.repend = req.delrepend = open + req.relrepend

	for look in self.nslist:
	    # absolute -> relative to start
	    look.reqtm = look.absreqtm - start
	    look.reptm = look.absreptm - start

        #
        # Patch for incorrect pers conn repends
        #
        try:
            for c in self.connlist:
                tl = c.translist
                if len(tl) > 1:
                    #print c.id
                    ti = 1
                    tr = tl[ti]
                    ltr = tl[0]
                    rqs = tr.reqstart
                    lastm = ltr.repend

                    for p in c.spktlist:
                        if p.len:
                            tm = p.tm
                            if rqs < tm:
                                #print ltr.repend/1000, '->', lastm/1000
                                ltr.repend = req.delrepend = lastm
                                ti += 1
                                if ti == len(tl):
                                    break
                                ltr = tr
                                tr = tl[ti]
                                rqs = tr.reqstart
                            lastm = tm
        except AttributeError, es:
            # catch inconsistencies
            
                str = 'TCPConn #%d repend adj: %s' % \
		      (c.id, es)
                if self.trace:
                    whoops(str)
                self.logfun(str)


#############################################################################
	    
    #
    # now that request/response timings have been made absolute can 
    # order the request lists
    #

    def order_translist(self):

	def sf(a, b):
	    if a.cvalid and b.cvalid:
		ret = a.reqstart - b.reqstart
	    elif a.cvalid:
		ret = a.reqstart - b.repstart
	    elif b.cvalid:
		ret = a.repstart - b.reqstart
	    elif a.svalid and b.svalid:
		ret = a.repstart - b.repstart
	    else:
		print 'Goof - two invalid operands to sort_translist_by_reqtm'

            try:
                return int(ret)
            except OverflowError:
                if ret > 0:
                    return 1
                elif ret < 0:
                    return -1
                else:
                    return 0

        trace = 1 and self.trace

        self.translist.sort(sf)

	# sanity
	if len(self.translist) and \
	   (self.translist[0].reqstart < self.connlist[0].open):
            str = 'WebClient #%s First request (%.3f) before first connection (%.3f)' % \
		  (self.addr_str(),
                   self.translist[0].reqstart/1000.0, 
		   self.connlist[0].open/1000.0)
            if trace:
                whoops(str)
            self.logfun(str)
	if len(self.translist) and \
           (self.translist[-1].repend > self.end): #self.translist[-1].TConn.close):
	    str = 'WebClient #%s WHOOPS last request after connection: (%.3f) (%.3f)' \
                  % (self.addr_str(),
                   self.translist[-1].repend/1000.0, 
		   self.translist[-1].TConn.close/1000.0)
            if trace:
                whoops(str)
            self.logfun(str)


#############################################################################

# 
# Build ordered list of significant events 

# - establishes ordering of events and allows time sequence gaps if wanted
#
# **idempotent**
#
    def build_event_list(self):

	def ev_cmp(a, b):
	    if a[0] > b[0]:
		return 1
	    elif a[0] < b[0]:
		return -1;
	    else:
		return a[1] - b[1]

	trace = 1 and self.trace
        
	reqs = self.translist
	conns = self.connlist
	lookups = self.nslist

	eventlist = self.eventlist
	mindur = 0x7FFFFFFFFFFFFFFFL
	maxdur = 0

	latestend = 0L
	for req in reqs:
	    cvalid = req.cvalid
	    svalid = req.svalid
	    if cvalid:
		reqstart = req.reqstart
		reqend = req.reqend
                delrepend = req.delrepend #
		reqdur = req.reqdur = int(reqend - reqstart)
		reqbegin = reqstart
		if not svalid:
		    reqfinish = reqend
		eventlist.append((reqstart, EV_REQSTART, req))
		eventlist.append((reqend, EV_REQEND, req))
		if reqend > latestend:
		    latestend = reqend
		if 0 < reqdur < mindur:
		    mindur = reqdur
	    else:
		req.reqdur = 0
	    if svalid:
		repstart = req.repstart
		repend = req.repend
		repdur = req.repdur = int(repend - repstart)
		if not cvalid:
		    reqbegin = repstart
		reqfinish = repend
		eventlist.append((repstart, EV_REPSTART, req))
		eventlist.append((repend, EV_REPEND, req))
                if req.repend != req.delrepend:
                    eventlist.append((req.delrepend, EV_DELREPEND, req))
                    repend = delrepend
                    repdur = int(delrepend - repstart)
		if repend > latestend:
		    latestend = repend
		if 0 < repdur < mindur:
		    mindur = repdur
	    else:
		req.repdur = 0

	    # spot case where order of req/rep may be compromised
	    if cvalid and svalid:
		if reqbegin > repstart:
		    reqbegin = repstart
		if reqfinish < reqend:
		    reqfinish = reqend

	    eventlist.append((reqbegin, EV_REQBEGIN, req))
	    eventlist.append((reqfinish, EV_REQFINISH, req))

	latest_connend = 0
	inter_pkt_mindur = 0x7FFFFFFFFFFFFFFFL

        # need distinct list of some connection events
        celist = self.celist
        
	for conn in conns:
	    cflags = conn.cflags
	    sflags = conn.sflags
	    optm = conn.open
	    clotm = conn.close

	    conndur = conn.dur = clotm - optm

	    if cflags & TSP_SYN:
	     celist.append((conn.clisyn, EV_CONN_CSYN, conn))

	    if sflags & TSP_SYN:
		celist.append((conn.servsyn, EV_CONN_SSYN, conn))

	    if cflags & TSP_FIN:
		celist.append((conn.clifin, EV_CONN_CFIN, conn))

	    if sflags & TSP_FIN:
		celist.append((conn.servfin, EV_CONN_SFIN, conn))

	    if cflags & TSP_ACKSYN:
		celist.append((conn.cliacksyn, EV_CONN_CACKSYN, conn))

	    if sflags & TSP_ACKSYN:
		celist.append((conn.servacksyn, EV_CONN_SACKSYN, conn))

	    if cflags & TSP_RST:
		celist.append((conn.clirst, EV_CONN_CRST, conn))

	    if sflags & TSP_RST:
		celist.append((conn.servrst, EV_CONN_SRST, conn))

	    celist.append((optm, EV_CONNOP, conn))
	    celist.append((clotm, EV_CONNCLO, conn))
	    

	    if clotm > latest_connend:
		latest_connend = clotm
	    if 0 < conndur < mindur:
		mindur = conndur

	    if (cflags & TSP_SYN) and (sflags & TSP_SYN) and \
	       0 < abs(conn.servsyn - conn.clisyn) < mindur:
		mindur = abs(conn.servsyn - conn.clisyn)

	    if conndur > maxdur:
		maxdur = conndur

	    #last_pkt_tm =  conn.pktlist[0].tm
	    last_pkt_tm =  conn.open
	    for pkt in conn.pktlist:
		tm = pkt.tm
		interpktdur = tm - last_pkt_tm
		if 0 < interpktdur < inter_pkt_mindur:
		    inter_pkt_mindur = interpktdur
		last_pkt_tm = tm
		#way = ord(pkt.way[0])
                way = pkt.dir
		if way == SERVER:
		    event = EV_CONN_SPKT