np_nretrans.py

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

#! /usr/bin/env python
###############################################################################
#                                                                             #
#   Copyright 2005 University of Cambridge Computer Laboratory.               #
#                                                                             #
#   This file is part of Nprobe.                                              #
#                                                                             #
#   Nprobe is free software; you can redistribute it and/or modify            #
#   it under the terms of the GNU General Public License as published by      #
#   the Free Software Foundation; either version 2 of the License, or         #
#   (at your option) any later version.                                       #
#                                                                             #
#   Nprobe is distributed in the hope that it will be useful,                 #
#   but WITHOUT ANY WARRANTY; without even the implied warranty of            #
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the             #
#   GNU General Public License for more details.                              #
#                                                                             #
#   You should have received a copy of the GNU General Public License         #
#   along with Nprobe; if not, write to the Free Software                     #
#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA #
#                                                                             #
###############################################################################


##############################################################################
## 
##
## How many sequence gaps or retransmissions on TCP connections
## 
##
## 

############################################################################ 

import string
import glob
import os
import sys
import nprobe
from sys import argv
import getopt
from signal import *

import Numeric

import np_file_util
import np_http_util
from np_obnode import *
from  np_longutil import *
from np_tfilter import *
from np_plot import *
from np_tcp import Tcp_Display

############################################################################
############################################################################

def usage(scriptname):
    print "usage: " + scriptname + "<rep-file-list>"

    sys.exit(1)

#############################################################################

#
# Aggregate cl list of connections wanted

def get_conns_wanted(s):
    connids = []
    for f in string.splitfields(s, ','):
	#print f
	#print '%d' % (string.atoi(f))
	connids.append(string.atoi(f))

    return connids

#############################################################################
	
def main():

    scriptname = 'np_nretrans.py'
    startconn = 0
    oneconn = None
    connstr = ''
    fspec = 0
    nwanted = 0
    nread = 0
    collect_gaps = 0


    try:
	optlist, args = getopt.getopt(sys.argv[1:], 'hr:s:c:F:g')

    except getopt.error, s:
	print '%s: %s' % (scriptname, s)
	usage(scriptname)
	sys.exit(1)

    print args
    if len(args) != 1:
	usage(scriptname)

    optstr = ''

    for opt in optlist:
	if opt[0] == "-h":
	    usage(scriptname)
	if opt[0] == "-r":
	    nwanted = string.atoi(opt[1])
	    optstr = optstr + '-r' + opt[1]
	if opt[0] == "-s":
	    startconn = string.atoi(opt[1])
	    optstr = optstr + '-s' + opt[1]
	if opt[0] == "-c":
	    oneconn = get_conns_wanted(opt[1])
	    optstr = optstr + '-c' + opt[1]
	    print 'analysing only connection(s) ',
	    print oneconn
	if opt[0] == "-F":
	    if opt[1] == 'h' or opt[1] == 'H':
		nprobe.filter_help()
		sys.exit(0)
	    fspec = string.atoi(opt[1])
	    optstr = optstr + '-F' + opt[1]
	if opt[0] == '-g':
	    collect_gaps = 1
	    gap_conns = []

    openfilelist, counters, basepath = np_file_util.get_files(args)
    

    #counters.printself("")
    filepath = basepath + 'counters'
    counters.printself_tofile(filepath, '')

    basepath = basepath + optstr

    nconns = 0
    nconns_accepted = 0
    # rejected by id filter
    id_rej = 0
    # rejected by connection filter
    cf_rej = 0
    # only one way seen
    one_ways = 0

    nread = 0
    ntrans = 0
    servtrans_bad = 0
    clitrans_bad = 0

    plain = 0
    rtg = 0
    nopens = 0

    c_ooo = 0
    n_c_ooo = 0
    s_ooo = 0
    n_s_ooo = 0
    c_dup = 0
    n_c_dup = 0
    s_dup = 0
    n_s_dup = 0
    c_gap = 0
    n_c_gap = 0
    s_gap = 0
    n_s_gap = 0

    if oneconn != None:
	nwanted = len(oneconn)


    done = 0
    for file in openfilelist:

	if len(openfilelist) > 1:
	    print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	    print "File %s - %d records" % (file.fnm, file.counters.nrecords)
	    print
	    sys.stdout.flush()
	while 1:
	    nread += 1
	    rectype = file.next_type_rec(REC_TCP_HTTP)
	    if rectype == -1: #EOF
		break

	    elif rectype & OPEN_BIT: # notification of TCP open
		file.advance()
		continue

	    elif rectype & HDRS_BIT: # a block of hdrs
		file.advance()
		continue

	    elif rectype == REC_TCP_HTTP: #TCP/HTTP
		#read the lot
		nconns += 1
		indx, connrec, translist =  np_http_util.get_http_rec(file)

	    else: # any other TCP close record
		# get tconn record and its data
		connrec = nprobe.tcp_conn()
		connrec.get_conn(file)
		print 'X?X?X?'
		file.advance()
		continue

	    # Have now got an HTTP close record - is it one we want?

	    # only want where both directions seen
	    if not (connrec.server_seen() and connrec.client_seen()):
		one_ways += 1
		continue

	    # don't want partial connections
	    state = connrec.get_flags()
	    if not(state & TCP_SERV_SYN) or not(state & TCP_CLI_SYN):
		nopens += 1
		continue

	    # get the conn_id
	    flow = connrec.flow_inner
	    if oneconn != None:
		conn_id = connrec.get_conn_id()
		if not oneconn.count(conn_id):
		    id_rej += 1
		    continue
		else:
		    print 'Got #%d' % (conn_id)
		    
	    elif id < startconn:
		continue
	    elif not nprobe.accept_conn(flow, fspec):
		cf_rej += 1
		continue

	    # 
	    # Now do what we're here for
	    #
	    nconns_accepted += 1
	    if nconns_accepted == nwanted:
		done = 1

	    if connrec.no_rtmts_or_gaps():
		plain += 1
		continue
	    else:
		rtg += 1
		n = connrec.n_c_ooo_pkts()
		if n:
		    c_ooo += 1
		    n_c_ooo += n
		n = connrec.n_s_ooo_pkts()
		if n:
		    s_ooo += 1
		    n_s_ooo += n
		n = connrec.n_c_dup_pkts()
		if n:
		    c_dup += 1
		    n_c_dup += n
		n = connrec.n_s_dup_pkts()
		if n:
		    s_dup += 1
		    n_s_dup += n
		n = connrec.n_c_gap_pkts()
		if n:
		    c_gap += 1
		    n_c_gap += n
		m = connrec.n_s_gap_pkts()
		if m:
		    s_gap += 1
		    n_s_gap += m

		if (m or n) and collect_gaps: 
		    gap_conns.append(connrec.get_conn_id())

	    if done:
		break
	if done:
	    break
		

    print 'Records read %d' % (nread)
    print 'Total HTTP connections %d' % (nconns)
    print 'Total HTTP connections accepted  %d' % (nconns_accepted)
    print '\t%d one way' % (one_ways)
    print '\t%d open not seen' % (nopens)
    print '\t%d rejected by flow id filter' % (id_rej)
    print '\t%d rejected by flow filter' % (cf_rej)
    print

    print 'No retransissions, oo pkts or seq gaps %d' % (plain)
    print 'With do %d' % (rtg)
    print '\tOO from client: %d pkts on %d connections' % (n_c_ooo, c_ooo)
    print '\tOO from server: %d pkts on %d connections' % (n_s_ooo, s_ooo)
    print '\tRT from client: %d pkts on %d connections' % (n_c_dup, c_dup)
    print '\tRT from server: %d pkts on %d connections' % (n_s_dup, s_dup)
    print '\tgaps from client: %d pkts on %d connections' % (n_c_gap, c_gap)
    print '\tgaps from server: %d pkts on %d connections' % (n_s_gap, s_gap)

    path = basepath + 'nrtmts'

    try:
	f = open(path, "w")
    except IOError:
	print 'Couldn\'t open file %s' % (path)
	sys.exit(1)

    f.write('Records read %d\n' % (nread))
    f.write('Total HTTP connections %d\n' % (nconns))
    f.write('Total HTTP connections accepted  %d\n' % (nconns_accepted))
    f.write('\t%d one way\n' % (one_ways))
    f.write('\t%d open not seen\n' % (nopens))
    f.write('\t%d rejected by flow id filter\n' % (id_rej))
    f.write('\t%d rejected by flow filter\n' % (cf_rej))
    print

    f.write('No retransissions, oo pkts or seq gaps %d\n' % (plain))
    f.write('With do %d\n' % (rtg))
    f.write('\tOO from client: %d pkts on %d connections\n' % (n_c_ooo, c_ooo))
    f.write('\tOO from server: %d pkts on %d connections\n' % (n_s_ooo, s_ooo))
    f.write('\tRT from client: %d pkts on %d connections\n' % (n_c_dup, c_dup))
    f.write('\tRT from server: %d pkts on %d connections\n' % (n_s_dup, s_dup))
    f.write('\tgaps from client: %d pkts on %d connections\n' % (n_c_gap, c_gap))
    f.write('\tgaps from server: %d pkts on %d connections\n' % (n_s_gap, s_gap))

    if collect_gaps and len(gap_conns):
	print 'Connections with sequence gaps:'
	f.write('Connections with sequence gaps:\n')
	for c in gap_conns:
	    print '\t#%d' % (c)
	    f.write('\t#%d\n' % (c))
	

	    

##############################################################################


# Call main when run as script
if __name__ == '__main__':
        main()