np_webhost.py

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

                    return
                if (not l.target) and (l.type & LR_INLINE):
                    l.target = req
                    req.reflink = l
                    if trace:
                        print '- in-line found (%d)' % (l.trans.order)
                    return

        # find latest followed link in links list
        def find_last_inline(req, links, refnode):
            ll = None
            for l in links:
                if l.tm > req.reqstart: # now too late
                    if trace:
                        print 'find_last_inline - balance of links too late'
                    break
                #print l
                if (l.type & (LR_INLINE | LR_FOLLOW_ON)) \
                   and (not l.target) and l.trans.uac == req.uac:
                    #print 'yes'
                    ll = l
            if ll:
                if not refnode:
                    #refnode = links[0].trans.ob
                    reftrans = ll.trans
                    refnode = reftrans.ob
                else:
                    reftrans = refnode.reqs[-1]
                #print 'll is'
                #print ll
                #print 'reftrans is', reftrans.order
                if ll.trans != reftrans:
                    add_referrer_link(req, reftrans, links, ll.type)
                    #if trace:
                        #raw_input('...')
                    #raw_input('...')
                else:
                    if trace:
                        print '- last inline found (%d)' % (ll.trans.order)
                    ll.target = req
                    req.reflink = ll
                if req.iscontainer and ll.trans.downlinks \
                   and req.reqstart - ll.trans.downlinks[-1].reqend > MAX_CONT_INLINE_TM:
                    ll.type &= ~LR_INLINE
                    ll.type |= LR_LINK
                    if trace:
                        print 'link converted to LINK'
            elif trace:
                print 'find_last_inline - no link found'
        
        # find earliest subsidiary of last in-line link in links list
        def find_inline_sub(self, req, links):
            if trace:
                print '- looking for multiple in-line fetch'
            ll = None
            for l in links:
                if l.tm > req.reqstart: # now too late
                    break
                if (l.type & LR_INLINE):
                    ll = l
            if ll:
                if trace:
                    print 'master found from %d' % (ll.trans.order)
                i = 0
                for sub in ll.subs:
                    if sub[0] > req.reqstart:
                        if trace:
                            print 'sub %d too late' % (i)
                        break
                    if trace:
                        print '- trying sub %d' % (i)
                    if not sub[2]:
                        if trace:
                            print 'got sub %d' % (i)
                        # not been followed 
                        dl = DummyLink(req, sub[0], ll.trans)
                        #dl.type = LR_INLINE_MULTI #sub[1]
                        req.reflink = dl
                        sub[2] = req

                        if sub[1] == LR_FOLLOW_ON:
                            # follow on for a fail
                            dl.type = LR_FOLLOW_ON
                            dl.trans = sub[3]
                            req.fon = 1
                            str = 'WebClient #%s Follow-on: (%d)' % \
                                  (self.addr_str(), req.order)
                            logfun(str)
                            if trace:
                                print '- follow on found (%d->%d)' % \
                                      (sub[3].order, req.order)
                        elif (sub[1] & LR_INLINE):
                            # simple in-line link
                            dl.type = LR_INLINE_MULTI
                            req.multi = 1
                            str = 'WebClient #%s Multiple fetch: trans %d' % \
                                  (self.addr_str(), req.order)
                            logfun(str)
                            if trace:
                                print '- multiple found (%d)' %  (req.order)
                                #
                        break
                    i += 1

        # find latest followed link in links list
        def find_link(req, links):
            ll = None
            for l in links:
                if l.tm > req.reqstart: # now too late
                    break
                if (not l.target) and l.trans.uac == req.uac:
                    ll = l
            if ll:
                req.reflink = ll
                ll.target = req
                if trace:
                    print 'find_link - type %x %s found (%d>)' % \
                          (ll.type, self.LinkDict.link_str(ll.type),
                           ll.trans.order)
            elif trace:
               print 'find_link - no link found' 

        # find latest in-line redirect link
        def find_inline_redirect(req, links):
            ll = None
            reqtm = req.reqstart
            if links[0].tm > reqtm:
                if trace:
                    print '- inline_redirect no timely link exists'
                return None
            for l in links:
                if l.tm > reqtm: # now too late
                    if trace:
                        print '- inline_redirect balance of links too late'
                    break
                    #return None
                if (not l.target) \
                   and l.type == LR_REDIRECT_INLINE:
                    #l.target = req
                    #req.reflink = l
                    if trace:
                        print '- in-line redirect found (%d)' %\
                              (l.trans.order)
                    ll = l
            return ll

                
        # find earliest in-line redirect link - request has referrer
        def find_inline_redirect_withref(req, links):
            reqtm = req.reqstart
            if links[0].tm > reqtm:
                if trace:
                    print '- redirect withref no timely link exists'
                return (None, None, None)
            rref = req.referrer
            la = None
            lc = None
            ls = None
            for l in links:
                if l.tm > reqtm: # now too late
                    if trace:
                        print 'redirect withref balance of links too late'
                    break
                if (not l.target) \
                   and l.type == LR_REDIRECT_INLINE:
                    if not lc and rref == l.trans.referrer:
                        lc = l
                    if not ls and rref == l.trans.absurl:
                        ls = l
                    if not la and not l.trans.referrer:
                        la = l
                if la and lc and ls:
                    break
                
            if trace:
                if la:
                    print '- in-line redirect (noref) found (%d)' % \
                          (la.trans.order)
                if lc:
                    print '- in-line redirect with common referrer found (%d)'\
                          % (lc.trans.order)
                if ls:
                    print '- in-line redirect as referrer found (%d)' % \
                          (ls.trans.order)
                if not (la or lc or ls):
                    print 'redirect withref - nothing found'
            return (la, lc, ls)

        # find latest redirect link in links list
        def find_link_redirect(req, links):
            ll = None
            for l in links:
                if l.tm > req.reqstart: # now too late
                    break
                if l.type == LR_REDIRECT_LINK:
                    ll = l
            if ll:
                #ll.target = req
                #req.reflink = ll
                if trace:
                    print '- redirect link found (%d)' % (ll.trans.order)
            elif trace:
                print '- redirect link none found'
            return ll

        # attempt to infer link in explicit referer by reference to neighbours
        def find_slot_referrer(req, refnode):
            gotit = None
            indx = req.order - 1
            for i in [indx-1, indx+1]:
                if not 0 <= i < len(self.translist):
                    continue
                t = self.translist[i]
                if t.reflink and t.uac == req.uac and t.referrer == req.referrer:
                    gotit = t
                    break

            if gotit:
                if trace:
                    print 'slot referrer found inferred from %d' % (i+1)

                # make a link
                #dl = DummyLink(req, req.reqstart, t.reflink.trans)
                dl = DummyLink(req, t.reflink.tm, t.reflink.trans)
                #dl.type = LR_REFERRER
                self.dummylinks.append(dl)
                refnode.ldict[req.absurl] = [dl.type, [dl]]
                req.reflink = dl

        #
        # FN STARTS HERE
        #
        
        trace = self.deps_trace
        logfun = self.logfun

        ldict = self.linkdict


        reqtm = req.reqstart
        reqobj = req.absurl
        referrer = req.referrer
        req.reflink = None


        if trace:
            print '******************************************************'
            print 'Finding referrer for %d %s uac %d' % (req.order,
                                                         reqobj, req.uac)

        #
        # Is this the result of an in-line re-direction
        # - check first as any referrer field is unlikely to point to
        #   correct transaction

        redlink = None
        try:
            f = ldict[reqobj]
            type = f[0]
            links = f[1]
            #if (type & LR_REDIRECT):
                #print 'redirect for '
            if (type & LR_REDIRECT) and (type & LR_INLINE):
                if trace:
                    print 'possible in-line redirect'
                if referrer:
                    la, lc, ls = find_inline_redirect_withref(req, links)
                    if ls:
                        redlink = ls # redirect is the referrer
                    elif lc:
                        redlink = lc # redirect has same referrer
                    else:
                        redlink = la # redirect has no referrer

                    if not redlink: # referrer claim lied
                        redlink = find_inline_redirect(req, links)
                else:
                    redlink = find_inline_redirect(req, links)

                # does it look like this is inline - 10s is arbitrary
                if redlink and req.reqstart - redlink.tm > 10000000:
                    if (type & LR_LINK):
                        # auto not worked - manual follow
                        if trace:
                            print 'auto redirect too late - looking for link'
                        redlink = find_link_redirect(req, links)
                    else:
                        redlink = None
                        if trace:
                            print 'auto redirect too late'
            else:
                # not inline
                redlink = find_link_redirect(req, links)
        except KeyError:
            pass

        if redlink:
            # got one
            redlink.target = req
            req.reflink = redlink
            if (not req.reflink.trans.reflink) and referrer and referrer != req.reflink.trans.absurl:
                # referrer hasn't a referrer and isn't this one's
                # - see if it can use this one's
                req = req.reflink.trans
                reqtm = req.reqstart
                reqobj = req.absurl
                if trace:
                    print 'attempting to find redirect\'s referrer using borrowed'
                    #raw_input('...')        
            else:
                # done all we can
                return

        # find links matching any referrer field
        if referrer:
            # have we seen the claimed referrer
            try:
                refnode = self.obdict[referrer]
                if trace:
                    if refnode.isdummy:
                        dstr = '(dummy)'
                    else:
                        dstr = ''
                    print '- referring node %s %s found' % (dstr, referrer)
                try:
                    f = refnode.ldict[reqobj]
                    type = f[0]
                    links = f[1]
                    firstlink = links[0]
                    # link there
                    if trace:
                        print 'link entry found'
                    if refnode.isdummy:
                        #
                        # need to check feasability of link from existing
                        # dummy trans
                        #
                        #print 'link uac', links[-1].target.uac
                        #print 'link uac', refnode.reqs[-1], refnode.reqs[-1].uac
                        #print 'req uac', req, req.uac
                        #print links
                        #if req.is_referrer():
                        if req.iscontainer:
                            # make a new timely dummy trans and link
                            #print 'referrer new dummy'
                            self.make_dummy_trans(req, referrer, links, refnode)
                        #elif links[-1].target.uac != req.uac:
                        elif refnode.reqs[-1].uac != req.uac:
                            #print 'uac new dummy'
                            self.make_dummy_trans(req, referrer, links, refnode) 
                        else:
                            #  XXX TODO test to avoid infeasably late linking
                            make_dummy_link(self, req, refnode)
                    elif firstlink.tm > reqtm:
                        if trace:
                            print 'too late', firstlink.tm, reqtm
                            print links
                        # but all fetches too late
                        if firstlink.type & DUM:
                            # earlist occurence already a dummy
                            make_link_earlier(firstlink, req)
                        else:
                            # make a timely dummy transaction
                            self.make_dummy_trans(req, referrer, links, refnode)
                    else:
                        # a candidate fetch exists
                        if (type & LR_INLINE):
                            find_last_inline(req, links, refnode)
                            if not req.reflink:
                                find_inline_sub(self, req, links)
                        else:
                            find_link(req, links)

                        if not req.reflink:
                            str = 'WebClient #%s Referring object links exhausted: %s->%s'% (self.addr_str(), referrer, reqobj)
                            logfun(str)
                            if trace:
                                inform(str)
                            if not req.iscontainer:
                                find_slot_referrer(req, refnode)


                except KeyError:
                    # no link recorded
                    if trace:
                        print 'no link entry found'
                    if refnode.isdummy:
                        #
                        # need to check feasability of link from existing