np_webhost.py

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

###############################################################################
#                                                                             #
#   Copyright 2005 University of Cambridge Computer Laboratory.               #
#                                                                             #
#   This file is part of Nprobe.                                              #
#                                                                             #
#   Nprobe is free software; you can redistribute it and/or modify            #
#   it under the terms of the GNU General Public License as published by      #
#   the Free Software Foundation; either version 2 of the License, or         #
#   (at your option) any later version.                                       #
#                                                                             #
#   Nprobe is distributed in the hope that it will be useful,                 #
#   but WITHOUT ANY WARRANTY; without even the implied warranty of            #
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the             #
#   GNU General Public License for more details.                              #
#                                                                             #
#   You should have received a copy of the GNU General Public License         #
#   along with Nprobe; if not, write to the Free Software                     #
#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA #
#                                                                             #
###############################################################################


from __future__ import generators

from print_col import *

#from urlparse import *
from nprobe import DUMMY_VALUE, LR_TS, LR_SCRIPTED, LR_ARCHIVE, LR_INLINE, \
     LR_LINK, LR_UNKNOWN, LR_REL_BASE, LR_END_BASE, LR_REDIRECT, \
     LR_SCRIPT_ARCHIVE, LR_SCRIPTED_LINK, LR_SCRIPTED_INLINE, \
     LR_SCRIPTED_UNKNOWN, LR_REL_SCRIPT_BASE, LR_END_SCRIPT_BASE, \
     LR_REDIRECT_INLINE, LR_REDIRECT_LINK, LR_REFRESH_URL, LR_REFRESH_SELF, \
     LR_REFRESH, LR_HAS_URL
from nprobe import TCP_CLIENT_SEEN, TCP_SERVER_SEEN
from nprobe import TCP_SERV_SYN, TCP_CLI_SYN, TCP_SERV_HTTP, SERVER, CLIENT, \
     TRANS_VAL, TRANS_DUMMY_UNSYNCH, TRANS_DUMMY_ERR, TRANS_ERR, \
     TRANS_INCOMPLETE, TRANS_FINISHED
from nprobe import TSP_SYN, TSP_DUP_SYN, TSP_FIN, TSP_ACKSYN, TSP_RST, \
     CT_NON_OBJECT, CT_DUMMY

from nprobe import linkrec
from nprobe import intoa_string, _free

from  np_longutil import *
from np_TCPConn import D_DUPCSYN, D_DUPFREQ, D_CRETRANS, D_NOT_CONNECTED, \
     D_REQNOTSEEN, D_REPNOTSEEN
from np_HTTPTrans import LinkRec, DummyTransaction
#from np_TCP import *
from print_col import *

from np_bitmaps import short_dash, short_dash_inv, med_dash, med_dash_inv

import sys
import scratch

##############################################################################
##############################################################################

#
# Help for TCP
#

TH_SYN = 0x2
TH_RST = 0x4
TH_FIN = 0x1

##############################################################################
##############################################################################

#
# Event types - ordering (for some) is important as used for sort
#


EV_NSOP = 10
EV_NSCLO = 20
EV_CONNOP = 30
EV_CONN_CPKT = 40
EV_CONN_CSYN = 45
EV_CONN_SPKT = 50
EV_CONN_SSYN = 55
EV_CONN_CACKSYN = 60
EV_CONN_SACKSYN = 70
EV_REQBEGIN = 80
EV_REQSTART = 90
EV_REQEND = 100
EV_REPSTART = 110
EV_REPEND = 120
EV_DELREPEND = 125
EV_REQFINISH = 130
EV_CONN_CFIN = 140
EV_CONN_SFIN = 150
EV_CONN_CRST = 160
EV_CONN_SRST = 170
EV_CONNCLO = 180

#############################################################################
#############################################################################

#
# Link types for non-parsed types (others defined in parse_object.h)
# - must all be > 0xffff
#

DUM =  0x10000 # dummy link
# mark dummy/referrer links as inline or link
#D_INLINE = 0x20000
D_INLINE = (DUM | LR_INLINE)
#D_LINK = 0x40000
D_LINK = (DUM | LR_LINK)

D_DTRANS_LINK = 0x20000 # link to unseen claimed referrer dummy transaction
D_DTRANS_INLINE = 0x40000 # inline do

D_FAIL = 0x80000 # dummy link for failed transaction

LR_MULTI =  0x100000 # multiple fetch
LR_INLINE_MULTI = (LR_MULTI | LR_INLINE)

LR_POSS_REVISIT = 0x200000
LR_REVISIT = (LR_POSS_REVISIT | LR_LINK)

LR_FAIL = (DUM | D_FAIL)
LR_DUMMY_INLINE = (DUM | D_INLINE)
LR_DUMMY_LINK = (DUM | D_LINK)

LR_FOLLOW_ON = (DUM | LR_INLINE | D_FAIL) # dummy link from failed transaction

# Links from dummy trans marked as inline, as always followed
LR_REFERRER_INLINE = (D_DTRANS_INLINE | D_INLINE)
LR_REFERRER_LINK = (D_DTRANS_LINK)

# Links to page roots
LR_ROOT = (LR_LINK | LR_REFRESH | D_DTRANS_LINK)

# Links which may indicate new container object
LR_NEW_CONT = (LR_ROOT | LR_REDIRECT)

#############################################################################
#############################################################################

#
# Cut-off periods for auto redirect/inline fetches
# - ARBITRARY
#

AUTO_REDIRECT_MAX_PER = 10*1000000
INLINE_MAX_PER = 60 * 1000000
INLINE_MAX_NOCONN = 60*1000000
MIN_LINK_TM = 2 * 1000000 # not feasible link click in less
MAX_CONT_INLINE_TM = 10*1000000
#############################################################################
#############################################################################


# Gathers all data relevant to a particular observed client/server pair

class WebHosts:
    
    def __init__(self, hostaddr, rooturl=None):

	self.Class = 'WebHosts'
	self.addr = hostaddr
        self.id = hostaddr
	self.iserver = 0

        # if rooturl distinguish ref tree with this root
        self.rooturl = rooturl

        # attachment point for NameCache
        #self.nslookup = lookup

        # lists populated during log scan
	self.connlist = []
        self.translist = []
	self.lookups = []

        self.rootlist = [] # page root objects
        self.unlinkedlist = [] # objects not linked into any tree
        self.refreshlist = [] # objects refreshed - not linked into trees
        self.unreslist = [] # s'be empty
        self.invalidlist = [] # invalid objects
        self.revisitlist = [] # revisited objects - not linked into trees

        self.obtree = None
        #self.obrec = FileRec(self.Class)
        self.tree_made = 0

        #np_glob.start = min(np_glob.start, self.start)

##     def __del__(self):
##         print 'Freeing WebClient'
##         for t in self.translist:
##             if t.linkstrlen:
##                 print 'freeing links'
##                 _free(t.linkstr)
##         print

#############################################################################

    def f_null(self, arg):

        pass

############################################################################# 

    def init_tree(self):

        # data structures populated when tree building
        self.nslist = [] # relevant dns lookups
        self.obdict = {} # objects seen
        self.linkdict = {} # global list of inline links of all types
	self.implied_obs = [] # dummy objects
        self.dummytrans = []
        self.dummylinks = []
        self.eventlist = []
        self.celist = []
        self.uconns = [] # not connected connections

	self.retcodedict = {}
	self.ctypedict = {}
	self.stypedict = {}
	self.uagentsdict = {}
	self.sagentsdict = {}
	self.errsdict = {}

        self.uagentcode = 0

        self.nlinks = 0
        self.nnolinks = 0
        self.trans_failed = 0
        self.trans_invalid = 0

#############################################################################

    def addr_str(self):

        #print self.addr
        return '%s/%s' % (intoa_string(self.addr[0]),
                          intoa_string(self.addr[1]))

#############################################################################

    def host_str(self):

        # just give the client address
        return intoa_string(self.addr[0])

############################################################################# 

    def make_tree(self, caller=None, tmbase=None, logfun=None, trace=0):
        
        if self.tree_made and trace == 0:
            # all done already
            return
        else:
            # either not done, or want it again with the cogs showing
            if self.tree_made:
                self.unmake_tree()
            self.tree_made = 1

        #print 'make_tree'
        #print_stack()

        if logfun == None:
            self.logfun = self.f_null
        else:
            self.logfun = logfun

        if trace:
            self.LinkDict = LinkDict()
            
        # just to record all seen
        self.logfun('WebClient #%s Seen:' \
                      % (self.addr_str()))
            
        self.caller = caller
        self.tmbase = tmbase
        self.trace = trace
        self.deps_trace = self.trace
        self.init_tree()
	self.order_connlist()
	self.build_nslist()
	self.adjust_tm_offsets()
	self.order_translist()
        self.scan_trans()
	self.build_event_list()
        self.find_curr_conns()
        self.find_unlinked()
        if trace and 0:
            for t in self.find_trees(): # just to show us
                pass
        if trace and 1:
            self.process_trees()


        scratch.conns_intrees += len(self.connlist)

        if trace and 0:
            print 'Opfr'
            for cop in self.opfr:
                i = 0
                for v in cop:
                    print i, v
                    i += 1
                print

############################################################################# 

    def unmake_tree(self):

        for c in self.connlist:
            c.lookup = None
            c.ns_attached = None

        del(self.nslist)

        for n in self.obdict.values():
            # includes dummy nodes
            del(n)
        del(self.obdict)

        for v in self.linkdict.values():
            for l in v[1]:
                del(l)
        del(self.linkdict)
        
        del(self.implied_obs)

        for t in self.dummytrans:
            del(t)
        del(self.dummytrans)

        for l in self.dummylinks:
            del(l)
        del(self.dummylinks)
        
        del(self.eventlist)
        del(self.celist)
        del(self.retcodedict)
        del(self.ctypedict)
        del(self.stypedict)
        del(self.uagentsdict)
        del(self.sagentsdict)
        del(self.errsdict)

#############################################################################

    def tidy_up(self):

        # free links bufs
        
        for cc in self.translist:
            if cc.linkstrlen:
                _free(cc.linkstr)

                
## ##                 l = cc.reflink
##                 if l:
##                     if l.target:
##                         del l.target
##                     l.target = None
##                     del cc.reflink
## ##                     del cc.ob
 ##            for ent in C.linkdict.values():
##                 for link in ent[1]:
##                     del link.target
##                     del link.trans

#############################################################################


    #
    # Order the connection list, find earliest open and latest close and delays
    #
    # **Idempotent**
    #

    def order_connlist(self):

        def sf(a, b):
            return a.id - b.id

        trace = 1 and self.trace

	tconns = self.connlist
	tconns.sort(sf)
	self.start = self.abstart = tconns[0].abstart

	i = 0
	end = tconns[0].abclose
	for c in tconns: