np_tcp.py

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

    basepath = basep
    
    if do_draw:
	logpath = '/dev/null'
    else:
	logpath = (basepath + '_machlog')

    #
    # Stats collector
    #
    invoke_args = ''
    for a in optlist:
        for aa in a:
            invoke_args += '%s' % (aa)
        invoke_args += ' '
    for a in args:
        invoke_args += '%s ' % (a)
        
    stats = TCPStats(invoke_args, quiet=quiet, logpath=logpath)
    stats.tfilt = tfilt
    stats.basepath = basepath

    #
    # Dictionary of tcp connections keyed by conn_id 
    # - used to collect headers
    #
    conn_ids = stats.conn_ids = {}

    #
    # Dictionary of objects 
    # - used to collect stats etc
    #
    obdict = stats.obdict = {}

    #
    # List of TCP connections found
    #
    conn_list = []

    #
    # Dict of servers seen
    #
    serv_dict = stats.serv_dict = {}

    #
    # Dict of clients seen
    #
    cli_dict = stats.cli_dict = {}

    #
    # Dict of agent types seen
    #
    adict = stats.adict = Atype()

    #
    # Bad analysis failure
    #
    bad_fails = []
    
    ectrs = stats.ectrs = E_Counters()

    nconns = 0
    # rejected by id filter
    id_rej = 0
    # rejected by connection filter
    cf_rej = 0
    # one way only
    ow_rej = 0
    # rejected by tfilter
    tf_rej = 0

    tcp_imps = TCP_Imps()

    nread = 0
    done = 0

    if do_draw:
        logfun = logfun_null
    else:
        logfun = stats.write_log

    #logfun = None

    gc.set_debug(gc.DEBUG_UNCOLLECTABLE)
    #gc.set_debug(gc.DEBUG_STATS)
    #print gc.get_threshold()
    gc.set_threshold(100, 10, 10)
    #raw_input('...')
    
    #
    # To get TCP records into
    #
    tcp_open_rec = tcp_open()
    hdrs = tcp_hdrs(1)
    connrec, translist = allocate_http_reusable_objects()

    for file in openfilelist:

        flen = os.stat(file.fnm)[6]

	if len(openfilelist) > 1:
	    print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	    print "File %s - %d records" % (file.fnm, file.counters.nrecords)
	    print
	    sys.stdout.flush()

	while not done:

	    if nrecs and nread > nrecs:
		break

	    rectype = file.next_type_rec(REC_TCP_HTTP)
            offset = file.curr_offset # get before read advances
            #print 'offset %d' % (offset)
            if offset >= flen:
                print 'ERROR bad offset %d - file size %d' % (offset, flen)
	    #print rectype
	    if rectype == -1: #EOF
		break

	    elif rectype & OPEN_BIT: # notification of TCP open
		tcp_open_rec.get_open_rec(file)
		id = tcp_open_rec.conn_id
		#print 'open %d' % (id)
                ectrs.nconns += 1
		if oneconn != None:
		    if oneconn.count(id):
			print 'got conn %d' % (id)
			conn_ids[id] = ([], []) #set up for list of headers
			#conn_ids_got += 1
		    else:
                        ectrs.id_rej += 1
			#print 'id_rej' 
		elif oneconn == None and id >= startconn:
		    if nprobe.accept_conn(tcp_open_rec.flow, fspec):
                        # set up for hdr and file/offset lists
			conn_ids[id] = ([], [])
		    else:
                        ectrs.cf_rej += 1
			#print 'cf_rej'
		else:
		    print 'REJECTED'
		
		continue

	    elif rectype & HDRS_BIT: # a block of hdrs
		#hdrs = nprobe.tcp_hdrs(0)
		hdrs.get_hdrs_p(file)
		conn_id = hdrs.conn_id
		if conn_ids.has_key(conn_id):
                    ent = conn_ids[conn_id]
                    ent[1].append((file.indx, REC_TCP_HTTP_HDRS, file, offset))
		    hlist = ent[0]
                    nhdr = len(hlist)-1
		    base_tm = ull2l(hdrs.get_hdrs_abstm())
                    for i in range(hdrs.nheld):
                        hlist.append(TCPHdr(hdrs, i, base_tm, nhdr))
                        nhdr += 1

		continue

	    elif rectype == REC_TCP_HTTP: #TCP/HTTP
                nread = nread + 1
		#read the lot
		#indx, connrec, translist =  get_http_rec(file)
                ntrans = get_http_rec_and_trans(file, connrec, translist)
                tlist = translist[:ntrans]

	    else: # any other TCP close record
		# get tconn record and its data
		connrec = nprobe.tcp_conn()
		connrec.get_conn(file)


	    # get the conn_id
	    conn_id = connrec.get_conn_id()

	    # only want where both directions seen
	    if not (connrec.server_seen() and connrec.client_seen()):
		if conn_ids.has_key(conn_id):
		    # remove dictionary item
		    del conn_ids[conn_id]
                    ectrs.ow_rej += 1
		    #print 'ow_rej'
		continue


	    if conn_ids.has_key(conn_id):
		#print 'got'
		if tfilt.pss(connrec, tlist): # one we're interested in


		    if oneconn != None and oneconn.count(id):
			print 'got conn %d' % (id)
			conn_ids_got += 1
			if conn_ids_got == len(oneconn):
			    print 'done'
			    done = 1

                    # find any headers already dumped
                    ent = conn_ids[conn_id]
                    ent[1].append((file.indx, REC_TCP_HTTP, file, offset)) 
		    hlist = ent[0]
		    #add any with the close record
		    base_tm = ull2l(connrec.get_hdrs_abstm())
                    nhdr = len(hlist)-1
                    for i in range(connrec.get_nhdrs_held()):
                        hlist.append(TCPHdr(connrec, i, base_tm, nhdr))
                        nhdr += 1
                    
                    # create Filerec for connection
                    fr = FileRec('TCPConn', ent[1])
                    #fr.add(ent[1])
                    obdict[conn_id] = fr
                    
		    # remove dictionary item
		    del conn_ids[conn_id]
		    #connrec.printself()

		    #conn.adjust_tm_offsets(conn.abstart)
                    #print 'start %.2f offset %.2f' % (conn.abstart/1000000.0, (conn.abstart-run_start)/1000000.0)

		    if do_draw:
			conn_list.append(fr)
			
		    else:
			#
			# run through the state machine
			#
                        conn = TCPConn(connrec, hlist, logfun, trace=trace)
                        conn.tlist = tlist
                        conn.adjust_tm_offsets(run_start)
                        conn.dur = conn.close - conn.open
                        conn.FileRec = fr
                        t = conn.model(tcp_imps, adict)
                        if t:
                            stats.add_TCPStats(t)
			# associate by server
			server = conn.server
			#print 'server %s' % (nprobe.intoa_string(server))
			if not serv_dict.has_key(server):
			    serv_dict[server] = []
			serv_dict[server].append(conn_id)
	## 		# associate by client
## 			client = conn.client
## 			#print 'client %s' % (nprobe.intoa_string(client))
## 			if not cli_dict.has_key(client):
## 			    cli_dict[client] = []
## 			cli_dict[client].append(conn_id)
			
		else:
		    # not interested - remove from dict
		    del(conn_ids[conn_id])
                    ectrs.tf_rej += 1
		    #print 'tfilt_rej'

            if not nread%100:
                overprint('Record %d' % (nread))

            if done:
                break
        
    overprint('%d relevant records' % (nread))

    # register this lot for future reference
    obdict['filepath'] = basepath
    obdict['stats'] = stats
    obdict['serv_dict'] = serv_dict

    #sys.exit(0)

    #
    # In drawing mode - fire up display and pass it connection list
    # - display will run conns through state machine so we can watch
    #
    if do_draw:
        ectrs.printself(tfilt)
        drawlist = []
        for c in conn_list:
            C = c.reconstruct(trace=1)
            C.adjust_tm_offsets(C.abstart)
            drawlist.append(C)
        TcpDisplay(drawlist, standalone='yes', path=basepath,
                   logfun=stats.nullf, trace=1)
	
    else:

        stats.results()
	    
	    
##############################################################################

def sort_a(a, b):
    return len(b[1])-len(a[1])
	    
##############################################################################


# Call main when run as script
if __name__ == '__main__':
        main()