np_webhost.py

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

		elif way == CLIENT:
		    event = EV_CONN_CPKT
		else:
		    print "GOOF - invalid way %d" % (way)
		    pkt.printself_rel()
		    sys.exit(1)
		eventlist.append((tm, event, pkt))
            del last_pkt_tm

        eventlist.extend(celist)
        #celist.sort(ev_cmp)

	if inter_pkt_mindur < mindur:
	    mindur = inter_pkt_mindur

	## print 'Mindur after pkts %s' % (tsLongstring(mindur))

## 	    print '#%2d %9d - %9d %9d (%9d)' % (conn.id, optm/1000, 
## 						    clotm/1000, 
## 						    (clotm-optm)/1000, 
## 						    latest_connend/1000)

	for ns in lookups:
	    reqtm = ns.reqtm
	    reptm = ns.reptm
	    dur = ns.dur = reptm - reqtm
	    
	    eventlist.append((reqtm, EV_NSOP, ns))
	    eventlist.append((reptm, EV_NSCLO, ns))

	    if 0 < dur < mindur:
		mindur = dur
	    if dur > maxdur:
		maxdur = dur
		
	#eventlist.sort(ev_cmp)
	eventlist.sort()
        
	if eventlist[0][1] != EV_CONNOP \
	   and eventlist[0][1] != EV_NSOP \
	   and eventlist[0][2].cobtype != CT_UNSYNCH \
	   and eventlist[0][2].cobtype != CT_POST_ERR \
	   and eventlist[0][2].sobtype != CT_UNSYNCH \
	   and eventlist[0][2].sobtype != CT_POST_ERR:
            str = 'WebClient #%s Event list goof:' % (self.addr_str())
            if trace:
                whoops(str)
                eventlist[0][2].printself()
            self.logfun(str)
	    #sys.exit(1)

	self.mindur = mindur
	self.maxdur = maxdur
	self.latest_reqend = latestend
	self.latest_connend = latest_connend


##############################################################################

#
# Establish # conns concurrently open
#
# cop1 spans open -> close (ie very first to very last pkt)
# cop2 spans open -> client/server FIN or client/server RST
# cop3 spans server ACKSYN -> client/server FIN or client/server RST
#
    def find_curr_conns(self):

        evl = self.celist
        evl.sort()
        #opfr = [[0]*10]*3
        opfr = [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]

        cop1 = 0
        cop2 = 0
        cop3 = 0

	maxcop1 = 0
	maxcop2 = 0
        maxcop3 = 0

        last_tm = evl[0][0]

        for e in evl:
	    tm = e[0]
	    type = e[1]
	    conn = e[2]
            id = conn.indx

            if last_tm > tm:
                print 'sort goof'
                sys.exit(1)
            last_tm = tm

            if type == EV_CONNOP: #always see this
                #print id, tm, 'op', cop1, cop2, cop3,
		# conn open
                conn.cop1 = cop1
                conn.cop2 = cop2
                conn.cop3 = cop3

                for i, n in ((0, cop1), (1, cop2), (2, cop3)):
                    try:
                        opfr[i][n] += 1
                    except IndexError:
                        for k in range(len(opfr[i]), n+1):
                            opfr[i].append(0)
                        opfr[i][n] += 1
                        
                conn.op = 1
                conn.op1 = 1
                conn.op2 = 0
                
                cop1 += 1
                cop2 += 1
                
		maxcop1 = max(maxcop1, cop1)
                maxcop2 = max(maxcop2, cop2)

                conn.close1 = conn.close
                #conn.close1 = max(conn.cldata, conn.sldata)

                #print '->', cop1, cop2, cop3
                
	    elif type == EV_CONNCLO: # always see this
                #print id, tm, 'clo', cop1, cop2, cop3,
		# conn close
                cop1 -= 1
                #conn.op = conn.op2 = 0
                if conn.op1:
                    cop2 -= 1
                    conn.op1 = 0
                    if conn.sdpkts:
                        conn.close1 = conn.sldata
                    elif conn.cdpkts:
                        conn.close1 = conn.cldata
                    else:
                        conn.close1 = conn.close
                        
                if conn.op2:
                    cop3 -= 1
                    conn.op2 = 0

                #print '->', cop1, cop2, cop3
                
	    elif type == EV_CONN_SACKSYN:
                #print id, tm, 'acksyn', cop1, cop2, cop3,
                cop3 += 1
                conn.op2 = 1
                maxcop3 = max(maxcop3, cop3)
                #print '->', cop1, cop2, cop3

            elif type == EV_CONN_CRST \
                              or type == EV_CONN_SRST \
                              or type == EV_CONN_CFIN \
                              or type == EV_CONN_SFIN:
                #print id, tm, 'end', type, cop1, cop2, cop3,
                if conn.op1:
                    cop2 -= 1
                    conn.op1 = 0
                if conn.op2:
                    cop3 -= 1
                    conn.op2 = 0
                #print '->', cop1, cop2, cop3
                conn.close1 = min(conn.close1, tm)

        self.maxcop1 = maxcop1
        self.maxcop2 = maxcop2
        self.maxcop3 = maxcop3
        self.opfr = opfr


##############################################################################

    # make dummy node, transaction and link for unseen explicit referrer
    def make_dummy_node(self, req, referrer):
        refnode = Obnode(referrer)
        refnode.isdummy = 1
        self.obdict[referrer] = refnode
        self.implied_obs.append(refnode)
        dt = DummyTransaction(req.referrer, refnode, req.reqstart,
                              req.connid, req.uac)
        dt.firstrans = req
        dt.order = -req.order
        self.dummytrans.append(dt)
        refnode.reqs.append(dt)
        refnode.dummy = dt
        dl = DummyLink(req, req.reqstart, dt)
        self.dummylinks.append(dl)
        refnode.ldict[req.absurl] = [dl.type, [dl]]
        req.reflink = (dl)


##############################################################################

    # make a dummy transaction for seen node
    def make_dummy_trans(self, req, referrer, links, refnode):
        if self.deps_trace:
            print '- making timely dummy >%d uac %d start %d' % \
                                      (req.order, req.uac, req.reqstart/1000)
        #dt = DummyTransaction(referrer, req.ob, req.repstart)
        dt = DummyTransaction(referrer, refnode, req.reqstart, req.connid,
                              req.uac)
        #print 'dummy trans made - reqstart=', dt.reqstart/1000
        dt.firstrans = req
        dt.order = -req.order
        dt.uac = req.uac ###
        self.dummytrans.append(dt)
        refnode.reqs.append(dt)
        dl = DummyLink(req, req.reqstart, dt)
        self.dummylinks.append(dl)
        if links:
            #links.insert(0, dl)
            links.append(dl)
        else:
            refnode.ldict[req.absurl] = [dl.type, [dl]]
        req.reflink = dl

        return dt

##############################################################################

    #
    # Find data dependencies for transactions
    #
    # Have to make some assumptions:
    # - if a referring object given this is `the truth' - unless redirected
    # - in-line objects are fetched following first unfollowed instance of a
    #   link to them
    # - other objects are fetched following last instance of link to them
    # - objects referred in-line and as followable links are got as in-line
    #
		    
    def find_deps(self, req):

        # make dummy link for dummy node
        def make_dummy_link(self, req, node):
            #dl = DummyLink(req, req.reqstart, node.dummy)
            #dl = DummyLink(req, req.reqstart, node.reqs[-1])
            trans = node.reqs[-1]
            if trans.isimplied:
                ltm = trans.repend
            else:
                ltm = (trans.repend - trans.repstart)/2
            dl = DummyLink(req, ltm, trans)
            #print 'trans=', trans.order, 'tm=', ltm
            self.dummylinks.append(dl)
            #node.reqs.append(req)
            node.ldict[req.absurl] = [dl.type, [dl]]
            req.reflink = (dl)
            if trace:
                print '- added link to dummy >%d' %\
                      (-node.reqs[-1].order)

        #move dummy link time forward
        def make_link_earlier(link, req):
            if trace:
                print '- modifying dummy >%d time' % \
                      (-link.trans.order)
            link.tm = req.reqstart
            link.target = req
            link.trans.repstart = req.reqstart
            link.trans.firsttrans = req
            link.trans.order = -req.order
            req.reflink = link

        # Provide link missing from referrer's links - first time link needed
        def insert_referrer_link(self, refnode, req):

            def by_tm(a, b):
                return int(a.tm-b.tm)

            ll = None # poss guide link
            gotone = 0
            #ft = refnode.reqs[0]
            #print 'reqstart', req.reqstart
            #for ft in refnode.reqs:
                #print ft.order, ft.repstart
            for ft in refnode.reqs:
                #print ft.order, ft.repstart, req.reqstart
                if ft.repstart > req.reqstart:
                    break
                if ft.uac == req.uac:
                    gotone = 1
            if gotone:
                if trace:
                    print '- likely transaction (%d) found' % (ft.order)
                # get ordered list of links in referrer for first fetch
                llist = []
                for l in refnode.ldict.values():
                    for ls in l[1]:
                        if ls.trans == ft:
                            llist.append(ls)
                if len(llist):
                    if trace:
                        print '- possible links found'
                    llist.sort(by_tm)
                    # find latest link giving rise to transaction earlier than this
                    for l in llist:
                        if l.trans.order < req.order:
                            ll = l
                if ll:
                    if trace:
                        print '- found guide link'
                    tm = ll.tm
                    ft = ll.trans ##
                else:
                    if trace:
                        print '- link not found - using repstart time'    
                    tm = ft.repstart
            else:
                tm = req.TConnopen
                if trace:
                    print '- no appropriate referrer fetch found, making dummy trans .%d' % (req.order)
                ft = DummyTransaction(refnode.absurl, refnode, tm, req.connid,
                                      req.uac)
                ft.firstrans = req
                ft.order = -req.order
                self.dummytrans.append(ft)
                ft.obnode = refnode
                refnode.dummy = ft
                refnode.reqs.insert(0, ft)

            # make a link
            dl = DummyLink(req, tm, ft)
            self.dummylinks.append(dl)
            refnode.ldict[req.absurl] = [dl.type, [dl]]
            req.reflink = dl
            if trace:
                print 'referrer link inserted'
                print dl

        def add_referrer_link(req, reftrans, links, type):
            #
            #  Provide link missing from referrer's links - link known to exist
            #  but not picked up for latest fetch of referrer
            #
            dl = DummyLink(req, reftrans.repstart, reftrans, type=type)
            self.dummylinks.append(dl)
            links.append(dl)
            req.reflink = dl
            if trace:
                print 'Adding missing link type %x to latest referrer fetch (%d)' \
                      % (dl.type, reftrans.order)



        #  NOT USED
        # give current transaction link from last for an object
        def grab_link(l, req, trace):
            tl = l.trans.downlinks
            tl.reverse()
            for t in tl:
                if t.ob == req.ob:
                    break
            tl.remove(t)
            tl.reverse()
            t.reflink = None
            req.reflink = l
            l.target = req
            if trace:
                print '- moved link from %d to %d' % (t.order, req.order)

        # NOT USED
        # object fetched but all links already followed - link to last fetch
        #  as a follow on
        def add_follow_on(self, l, req, trace, links):
            #print 'add_follow_on'
            if l.target.absurl != req.absurl:
                print 'OUCH %s != %s' % (l.target.absurl, req.absurl)
            
            dl = DummyLink(req, l.target.reqend, l.target, type= LR_FOLLOW_ON)
            self.dummylinks.append(dl)
            print l.trans.ob
            
            try:
                l.trans.ob.ldict[req.absurl][1].append(dl)
                print 'dict OK'
            except KeyError:
                print 'FOO Key error'
            except IndexError:
                print 'FOO Index error'
            except:
                print 'FOO other error'
            req.reflink = dl
            if trace:
                print '- added follower link from %d to %d' % (l.target.order, req.order)
                
                        
        # find earliest followed in-line link in links list
        def find_inline(req, links):
            for l in links:
                if l.tm > req.reqstart: # now too late