np_webhost.py

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

	    c.indx = i
	    c.ns_attached = 0
	    i += 1
            end = max(end, c.abclose)
                

	self.end = end

#############################################################################


    #
    # Order the links dictionary transaction lists 
    #

    def order_linksdicts(self):

        def by_tm():
            return int(a.tm - b.tm)

        for l in self.linkdict.values():
            l[1].sort(by_tm)

        for ob in self.obdict.values():
            if ob.is_referrer:
                for l in ob.ldict.values():
                   l[1].sort(by_tm) 

#############################################################################

    #
    # Main loop through all the transactions seen
    #

    def scan_trans(self):

        # for sorting rootlist - abs() because dummy trans orders are < 0
        def by_req(a, b):
            return abs(a.order) - abs(b.order)

        trace = 1 & self.trace
        if trace:
            print 'Scanning trans for client', self.addr_str()

        obdict = self.obdict
        ctypedict = self.ctypedict
        stypedict = self.stypedict
        uagentsdict = self.uagentsdict
        sagentsdict = self.sagentsdict
        errsdict = self.errsdict
        retcodedict = self.retcodedict

        nerrs = 0
        
        n = 1 # start at 1 so any dummy for trans can be differentuated by neg.

        last_trans = None
        curr_referrer = None

        for t in self.translist:

            t.order = n
            n += 1
            t.reflink = None
            t.downlinks = []
            host = t.host

            absurl = t.absurl
            
            #
            # Make node dictionary entry for each container object seen
            #

            if obdict.has_key(absurl):
                node = obdict[absurl]
                if node.isdummy:
                    # replace with a real one
                    dummy = node
                    node = Obnode(absurl)
                    obdict[absurl] = node
                    node.dummy = dummy
                t.ob = node
                t.iscontainer = 1
                    
            elif t.iscontainer:
                node = Obnode(absurl)
                obdict[absurl] = node
                t.ob = node

            else:
                t.ob = None

                
            # record any links in object
            got_redirect = t.get_links()

            if t.linkstrlen:
                node.reqs.append(t)


            #
            # Maintain dictionaries of agents, object types, etc
            #
            if t.cvalid:
                obtype = t.cobtype
                uagent = t.uagent
                err = t.cerr

                # maintain integer mapping of user agents for quick comparison
                try:
                    ua = uagentsdict[uagent]
                except KeyError:
                    ua = [self.uagentcode, 0]
                    uagentsdict[uagent] = ua
                    self.uagentcode += 1

                ua[1] += 1
                t.uac = ua[0]

                if err:
                    if not errsdict.has_key(err):
                        errsdict[err] = 1
                    else:
                        errsdict[err] += 1

            else:
                obtype = CT_NON_OBJECT
                t.uac = -1

            if not ctypedict.has_key(obtype):
                ctypedict[obtype] = 1
            else:
                ctypedict[obtype] += 1


            need_redirect = 0

            if t.svalid:
                obtype = t.sobtype
                retcode = t.retcode
                server = t.server
                err = t.serr

                if server == None:
                    server = 'Unknown'
                if not sagentsdict.has_key(server):
                    sagentsdict[server] = 1
                else:
                    sagentsdict[server] += 1

                if err:
                    if not errsdict.has_key(err):
                        errsdict[err] = 1
                    else:
                        errsdict[err] += 1

                if not retcodedict.has_key(retcode):
                    retcodedict[retcode] = 1
                else:
                    retcodedict[retcode] += 1

                if retcode/100 == 3 and retcode != 304:
                    need_redirect = 1
                    self.logfun('WebClient #%s Redirection:' % (self.addr_str()))
                    #print 'need redirect',
                    #if got_redirect:
                        #print ' got it'
                    #else:
                        #print 'not got'
                    #print '%d got_redirect = %x' % (t.order, got_redirect)

            else:
                obtype = CT_NON_OBJECT

            if not stypedict.has_key(obtype):
                stypedict[obtype] = 1
            else:
                stypedict[obtype] += 1

            if not t.cvalid:
                self.invalidlist.append(t)
                continue

            if need_redirect and not got_redirect:
                t.isfail = 1
                str = 'WebClient #%s No redirect URL: (%d)' % \
                      (self.addr_str(), t.order)
                self.logfun(str)
                if trace:
                    inform(str)
            if need_redirect and got_redirect:
                t.isfail = 0 # over-rides finished/complete as measure of succes
            # fit into reference tree
            t.multi = 0
            t.fon = 0
            self.find_deps(t)
            reflink = t.reflink
            if not reflink and last_trans and t.order < len(self.translist):
                # can we infer from previous?
                if (last_trans.referrer == self.translist[t.order].referrer \
                    or self.translist[t.order].referrer == t.absurl) \
                   and last_trans.reflink:
                    t.referrer = last_trans.reflink.trans.absurl
                    if trace:
                        print 'trying again with referrer', t.referrer
                    self.find_deps(t)
                    
            if reflink:
                self.nlinks += 1
                #t.reflink.trans.downlinks.append(t)
                if t.isfail:
                  self.register_fail(t, trace)
                        
            else:
                self.nnolinks +=1

            if t.isfail:
                self.trans_failed += 1
            #
            # Is this a persistent connection *with* multiple fetches
            #
            if t.connorder == 1:
                #t.TConn.persist = 1
                str = 'WebClient #%s Persistent connection(s):' % \
                      (self.addr_str())
                self.logfun(str)
                if trace:
                    inform(str)

            #
            # Is this a root etc
            #
            if reflink == None:
                    self.unlinkedlist.append(t)
            else:
                type = reflink.type
                if type & LR_ROOT:
                    if type & LR_REFRESH:
                        self.refreshlist.append(t)
                    else:
                        self.rootlist.append(t)
                elif not type & LR_INLINE:
                    self.unreslist.append(t)

            #
            # Check for late in-line fetches not descendent of current
            # container - result of reloads for back button?
            #
            if reflink:
                ltype = reflink.type
                reftrans = reflink.trans
                if not curr_referrer:
                    # first
                    curr_referrer = reftrans
                    if trace:
                        print 'new referrer', curr_referrer.order
                elif reftrans.isimplied:
                    if trace:
                        print 'new referrer', curr_referrer.order
                    curr_referrer = reftrans
                elif reftrans == last_trans:
                    
                    if not last_trans.reflink and last_trans.iscontainer:
                        curr_referrer = last_trans
                        if trace:
                            print 'new referrer', curr_referrer.order
                    else:
                        last_trans.reflink.type &= ~LR_POSS_REVISIT
                        if last_trans.reflink.type & LR_NEW_CONT:
                            curr_referrer = last_trans
                            if trace:
                                print 'new referrer', curr_referrer.order
                        
             ##    elif reftrans != curr_referrer \
##                          and not t.iscontainer \
##                          and not reftrans.isimplied \
##                          and not ltype & LR_REDIRECT:
##                     reflink.set_type(LR_REVISIT)
##                     self.revisitlist.append(t)
##                     str = 'WebClient #%s Link -> revisit - not current referrer:' % \
##                           (self.addr_str())
##                     self.logfun(str)
##                     if trace:
##                         inform(str)

                #
                # Infeasably long fetch times - arbitrary period
                #  ? result of reload - assume re-visit
                #
                try:
                    deadper = t.TConn.open - reftrans.downlinks[-1].TConn.close
                    #llink = reftrans.downlinks[-1].reflink.type
                    #print t.TConn.open, t.reflink.trans.downlinks[-1].TConn.close
                    #print 'conntm', deadper
                except (IndexError, AttributeError):
                    deadper = 0
                    #llink = 0x0
                #print 'deadper=', deadper/1000, 'reqstart=', t.reqstart/1000, 'reflink tm=', reflink.tm/1000
                #if llink & LR_LINK \
                   #or \
##                 if (deadper > INLINE_MAX_NOCONN and t.reqstart - reflink.tm > INLINE_MAX_PER) \
##                    or (not deadper \
##                        and t.reqstart - reflink.tm > INLINE_MAX_PER):
                #if deadper > INLINE_MAX_NOCONN:
                    # assume this is a revisit
                    #print 'revisit?', t.order
                    #if not (t.reflink.type & LR_LINK):
                if (deadper > INLINE_MAX_NOCONN
                    or t.reqstart - reflink.tm > INLINE_MAX_PER) \
                    and (ltype & LR_INLINE):
                    #reflink.type = LR_REVISIT
                    #if not t in self.revisitlist:
                        #self.revisitlist.append(t)
##                         # print 'yes'
                    str = 'WebClient #%s Link -> revisit - time:' % \
                          (self.addr_str())
                    self.logfun(str)
                    if trace:
                        inform(str)
                        
                    refnode = reflink.trans.ob
                    if refnode: 
                        if t.absurl in refnode.ldict:
                            links = refnode.ldict[t.absurl]
                        else:
                            links = None
                        dt = self.make_dummy_trans(t, reflink.trans.absurl,
                                           links, refnode)
                        reflink.target = dt
                        dt.reflink = reflink
                        reflink.type = (LR_REVISIT | LR_LINK)
                        reftrans.downlinks.append(dt)
                        reftrans = dt
                        self.rootlist.append(dt)
                    else:
                        str = 'WebClient #%s Link -> no reflink.trans.ob:' % \
                              (self.addr_str())
                        self.logfun(str)
                        if trace:
                            inform(str)
                        
                reftrans.downlinks.append(t)
            if reflink and not (ltype & LR_POSS_REVISIT):       
                last_trans = t


            
        # add in dummy transactions for inlines - must be roots
        for d in self.dummytrans:
            for t in d.downlinks:
                if t.reflink.type & LR_INLINE:
                    self.rootlist.append(d)
                    break

        # have any unlinked or refreshed now become roots?
        ull = self.unlinkedlist
        for i in range(len(ull)):
            t = ull[i]
            if t.downlinks or t.reflink:
                ull[i] = None
                self.rootlist.append(t)
        self.unlinkedlist = [t for t in ull if t != None]
        
        rl = self.refreshlist
        for i in range(len(rl)):
            t = rl[i]
            if t.downlinks:
                rl[i] = None
                self.rootlist.append(t)
        self.refreshlist = [t for t in rl if t]
        
        il = self.invalidlist
        for i in range(len(il)):
            t = il[i]
            if t.downlinks:
                il[i] = None
                self.rootlist.append(t)
        self.invalidlist = [t for t in il if t]
                
        self.rootlist.sort(by_req)
        

#############################################################################

    #
    # To allow repeat requests to be linked from failures add an entry to
    #  the failed tran's downlink subsidiary links (where find_dep for the
    #  repeat will find it)
    #  - additional field [3] gives failed trans
    #

    def register_fail(self, trans, trace):