np_httptrans.py

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

                    
		if type == LR_TS:
                    #del(rec)
		    continue
		elif (type &  LR_REL_BASE):
		    #print "Base PUSH"
                    base = link.url
                    u = list(urlparse(base, 'http'))
                    h = u[1].split(':')
                    if len(h) != 2:
                        # no port no.
                        ps = ':80'
                        u[1] += ps
                    else:
                        ps = h[1]
                    base = urlunparse(u)
		    bstack.append((base, ps))
                    bslen += 1
                    #del(rec)
		    continue
		elif (type &  LR_END_BASE):
		    #print "Base POP"
                    if bslen > 1:
                        bstack.pop()
                        bslen -= 1
                    else:
			str = 'TCPConn #%d Base stack empty:' % (self.connid)
                        if self.trace:
                            whoops(str)
                            #conn.tconn.printself()
                            #trans.printself(conn.tconn)
                        self.logfun(str)
		    base, ps = bstack[-1]
                    #del(rec)
		    continue
		#print '%s %d %x' % (rec.url, rec.tm, rec.type)
		#raw_input('anything to continue...\n')

                # must be a link/inline/unknown URL
                self.ob.is_referrer += 1

                url = urljoin(base, link.url)
                if not url:
                    continue
                p = urlparse(url)
                if len(p[1].split(':')) != 2:
                    # no port no.
                    p = list(p)
                    p[1] += ps
                    url = urlunparse(p)
                if not p[2]:
                    p = list(p)
                    p[2] = '/'
                    url = urlunparse(p)
                    
                rec = LinkRec(link)
                rec.trans = self
                rec.tm = rec.reltm + self.repstart
                rec.url = url

                if type & LR_REFRESH:
                    if type == LR_REFRESH_SELF:
                        url = self.absurl
                    type = LR_REFRESH
                    rec.type = LR_REFRESH
                    self.logfun('WebClient #%s Refresh: %s' % \
                   (self.client.host_str(), url))

                if gotr and url == rurl:
                    type = rec.type = LR_REDIRECT_LINK
                    
                if (type & LR_REDIRECT):
                    gotr = 1
                    rurl = url
                    
		#print '%s\n%s\n%s\n\n' % (base[-1], rec.url, url)
		#raw_input('anything to continue...\n')

                #
                # This is a PATCH - *.css shows wrong link type
                #
                is_css = 0
                if url[-4:] == '.css':
                    type &= ~LR_LINK
                    type |= LR_INLINE
                    rec.type = type
                    #is_css = 1


                subd = 0
                for dict in (self.client.linkdict, self.ob.ldict):
                    try:
                        f = dict[url]
                        f[0] |= type
                        if f[1][-1].trans == self \
                           and f[1][-1].type == type:
                            if not subd:
                                # add subsidiary record
                                # fields are link-seen time, link type and target
                                f[1][-1].subs.append([rec.tm, type, None])
                                subd = 1
                                if is_css:
                                    print 'sub added', rec
                                #raw_input('...')
                        else:
                            f[1].append(rec)
                            rec.subs = []
                            rec.target = None
                            rec_used = 1
                            if is_css:
                                print 'another added', rec
                                #raw_input('...')
                            #if type & LR_REDIRECT:
                                #print 'TCPConn #%d adding indirect %d to %s' % (self.order, len(f[1]), url)
                    except KeyError:
                        dict[url] = [type, [rec]]
                        if is_css:
                            print 'new added', rec
                            #raw_input('...')
                        rec.target = None
                        rec.subs = []
                        rec_used = 1
                        #if type & LR_REDIRECT:
                            #print 'TCPConn #%d adding first indirect to %s' % (self.order, url)

##                 if is_css:
##                     print self.ob.ldict
##                     raw_input('...')

	    
	else:
	    # not svalid
	    self.relrepstart = self.relrepend = self.relreqend
	    self.links = {}
            
            

        #print 'returning %x' % (gotr)
        
        return gotr
        
#############################################################################

    def is_container(self):
        """
        is_container(self) - does the object delivered contain links, etc
        
        """
        
        return self.ob.is_referrer and self.retcode == 200 and not self.isfail
        
#############################################################################

    def is_referrer(self):
        """
        is_referrer(self) - is this an object which might contain links
        """
        # XXX TODO - need better test - eg. could be a frame
        return self.ob.is_referrer \
           or (self.retcode != 200 and self.sobtype == 1)


        
#############################################################################


    def printself(self):
	self.trans.printself(self.conn.tconn)
        
#############################################################################
	

    def printself_long(self):
	self.trans.printself(self.conn.tconn)
	print '***************************************************************'
	self.conn.tconn.printself()
	for t in self.conn.translist:
	    t.printself(self.conn.tconn)
        
#############################################################################

    def print_links(self):
        pass

## 	def sort_link_by_time(a, b):
## 	    return int(a[1][0] - b[1][0])

## 	print 'Object %s' % (self.absurl)

## 	linklist = self.inlines.items()
## 	linklist.sort(sort_link_by_time)
## 	print 'In-line:-'
## 	for link in linklist:
## 	    print '%s (x%d) %s' % (link[0], link[1][1], 
## 				   tsLongstring(link[1][0]))

## 	linklist = self.links.items()
## 	linklist.sort(sort_link_by_time)
## 	print 'Links:-'
## 	for link in linklist:
## 	    print '%s (x%d) %s' % (link[0], link[1][1], 
## 				   tsLongstring(link[1][0]))

## 	linklist = self.unknowns.items()
## 	linklist.sort(sort_link_by_time)
## 	print 'Unknowns:-'
## 	for link in linklist:
## 	    print '%s (x%d) %s' % (link[0], link[1][1], 
## 				   tsLongstring(link[1][0]))

	
	
#############################################################################

    def f_null(self, arg):

        pass

#############################################################################
#############################################################################

class DummyTransaction(Transaction):
    

    def __init__(self, obname, ob, time, connid, uac):
	self.obname = obname
        self.ob = ob
        self.uac = uac
        ob.is_referrer = 1
        self.iscontainer = 1

        # XXX TODO adjust for intervening RTT's
	self.reqstart = self.reqend = self.repstart = self.repend = time
        
	self.connid = -connid
        self.TConn = None
	self.cvalid = 1
	self.svalid = 1
	self.followed = []
	self.inlines = {}
	self.links = {}
	self.unknowns = {}
        self.isimplied = 1
        self.servaddr = None
        self.absurl = ob.absurl
        self.sobtype = CT_DUMMY

        self.nbytes = 0
        self.hdrlen = 0

        self.reflink = None
        self.downlinks = []
	return

	
	
#############################################################################
#############################################################################

## class LinkRec(linkrec):

##     def __init__(self):
        
##         linkrec.__init__(self)
##         self.trans = None
##         self.target = None
##         self.subs = None

##     def get_url(self):

##         return self.url

    
##     def __repr__x(self):

##         from np_WebHost import LinkDict
##         link_str = LinkDict().link_str

##         str = 'LinkRec type %x %s\n' % (self.type,
##                                         link_str(self.type))
##         str += 'URL: %s\n' % (self.get_url())
##         if hasattr(self, 'trans'):
##             str+= ' Referrer %d' % (self.trans.order)
##         #if hasattr(self.target, 'order'):
##             #str += ' Target %d' % (self.target.order)
##         if self.target:
##             str += ' Target %d' % (self.target.order)
##         else:
##             str += ' No target'
##         if self.type & LR_REFRESH:
##             str += ' Period: %d\n' % (self.per)
##         if hasattr(self, 'tm'):
##             str += ' Abs tm %ld.%.6ldus\n' % (self.tm/1000000, self.tm%1000000)
##         str += ' Rel tm %ld.%.6ldus\n' % (self.reltm/1000000,
##                                              self.reltm%1000000)
##         if self.subs:
##             print 'Subs', self.subs

##         return str

	
	
#############################################################################
#############################################################################

class LinkRec:

    def __init__(self, lr):
        
        self.trans = None
        self.target = None
        self.subs = None

        if lr:
            self.reltm = lr.reltm
            self.type = lr.type
            self.per = lr.per

    def get_url(self):

        return self.url

    
    def __repr__x(self):

        from np_WebHost import LinkDict
        link_str = LinkDict().link_str

        str = 'LinkRec type %x %s\n' % (self.type,
                                        link_str(self.type))
        str += 'URL: %s\n' % (self.get_url())
        if hasattr(self, 'trans'):
            str+= ' Referrer %d' % (self.trans.order)
        #if hasattr(self.target, 'order'):
            #str += ' Target %d' % (self.target.order)
        if self.target:
            str += ' Target %d' % (self.target.order)
        else:
            str += ' No target'
        if self.type & LR_REFRESH:
            str += ' Period: %d\n' % (self.per)
        if hasattr(self, 'tm'):
            str += ' Abs tm %ld.%.6ldus\n' % (self.tm/1000000, self.tm%1000000)
        str += ' Rel tm %ld.%.6ldus\n' % (self.reltm/1000000,
                                             self.reltm%1000000)
        if self.subs:
            print 'Subs', self.subs

        return str
	
	
#############################################################################
#############################################################################