np_sperf.py

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

#! /usr/bin/env python
###############################################################################
#                                                                             #
#   Copyright 2005 University of Cambridge Computer Laboratory.               #
#                                                                             #
#   This file is part of Nprobe.                                              #
#                                                                             #
#   Nprobe is free software; you can redistribute it and/or modify            #
#   it under the terms of the GNU General Public License as published by      #
#   the Free Software Foundation; either version 2 of the License, or         #
#   (at your option) any later version.                                       #
#                                                                             #
#   Nprobe is distributed in the hope that it will be useful,                 #
#   but WITHOUT ANY WARRANTY; without even the implied warranty of            #
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the             #
#   GNU General Public License for more details.                              #
#                                                                             #
#   You should have received a copy of the GNU General Public License         #
#   along with Nprobe; if not, write to the Free Software                     #
#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA #
#                                                                             #
###############################################################################


##############################################################################
## 
##
## 
## 
##
## 

############################################################################ 

import string
import glob
import os
import sys
import nprobe
from sys import argv
import getopt
from signal import *

import Numeric

## import np_file_util
## import np_http_util
## from np_obnode import *
## from  np_longutil import *
## from np_tfilter import *
## #from np_plot import *
## #from np_plot import DataSet, DATA_TS, EmptyDataSetError
## import np_plot


import np_file_util
import np_http_util
import np_ns_utils
from np_obnode import *
#from np_draw_obtree import *
from np_TCP_Window import *
from  np_longutil import *
from np_lookups import *
from np_tfilter import *
from np_TCP import *
from TCP_Imp import *
from np_atypes import Atype

from Tkinter import *
from np_widgets import *

from np_notesel import *

############################################################################
############################################################################

EV_CONN_SOUGHT = 0
EV_CONN_RESP = 1
EV_OB_REQ = 2
EV_OB_START = 3
EV_OB_SERVED = 4
EV_CONN_CLOSE = 5
EV_DUP_SYN = 6
EV_RTMTS = 7

P_DS = 8
P_RTMT = 9

############################################################################
############################################################################

def usage(scriptname):
    print "usage: " + scriptname + '<-p/P(eriod s)>' + " <rep-file-list>"
    print '\t-p -- aggregate as occurences per specified aggregation period'
    print '\t-P -- aggregate as occurences per second over aggregation period'

    sys.exit(1)

#############################################################################

#
# Aggregate cl list of connections wanted

def get_conns_wanted(s):
    connids = []
    for f in string.splitfields(s, ','):
	#print f
	#print '%d' % (string.atoi(f))
	connids.append(string.atoi(f))

    return connids

#############################################################################

def handle_sigint(n, f):

    global logf

    print 'SIGINT'
    try:
	cache_lookups()
	if logf != None:
	    logf.flush()
    except:
	    sys.exit(0)

#############################################################################
	
def main():

    scriptname = 'np_sperf.py'
    nrecs = 0
    startconn = 0
    oneconn = None
    connstr = ''
    fspec = 0
    per = 1.0
    per_sec = 0


    try:
	optlist, args = getopt.getopt(sys.argv[1:], 'hvr:s:c:F:T:p:P:')

    except getopt.error, s:
	print scriptname + ": " + s
	usage(scriptname)
	sys.exit(1)

##     print args
##     if len(args) != 1:
## 	print 'foo'
## 	usage(scriptname)

    tfilt = TFilter()

    optstr = ''

    for opt in optlist:
	if opt[0] == "-h":
	    usage(scriptname)
	if opt[0] == "-p":
	    per = string.atof(opt[1])
            optstr = optstr + '-p' + opt[1]
	if opt[0] == "-P":
	    per = string.atof(opt[1])
            optstr = optstr + '-P' + opt[1]
	    per_sec = 1
	if opt[0] == "-v":
	    nprobe.set_print_full_hostname(1)
	if opt[0] == "-r":
	    nrecs = string.atoi(opt[1])
	    optstr = optstr + '-r' + opt[1]
	if opt[0] == "-s":
	    startconn = string.atoi(opt[1])
	    optstr = optstr + '-s' + opt[1]
	if opt[0] == "-c":
	    oneconn = get_conns_wanted(opt[1])
	    optstr = optstr + '-c' + opt[1]
	    print 'analysing only connection(s) ',
	    print oneconn
	if opt[0] == "-F":
	    print 'foo'
	    if opt[1] == 'h' or opt[1] == 'H':
		nprobe.filter_help()
		sys.exit(0)
	    fspec = string.atoi(opt[1])
	    optstr = optstr + '-F' + opt[1] 
	if opt[0] == "-T":
	    if opt[1] == 'h' or opt[1] == 'H':
		tfilt.help()
		sys.exit(0)
	    tfilt.build_filters(opt[1])
	    optstr = optstr + '-T' + opt[1]

    if per == 1.0:
	print 'No aggregation period specified - defaulting to %.3fs' % (per)


    signal(SIGINT, handle_sigint)

    openfilelist, counters, basepath = np_file_util.get_files(args)
    

    #counters.printself("")
    filepath = basepath + 'counters'
    counters.printself_tofile(filepath, '')

    basepath = basepath + optstr

##     #
##     # Dictionary of tcp connections keyed by conn_id 
##     # - initially used to collect headers, then to reference connections
##     #
##     conn_ids = {}

    #
    # event lists
    #
    evlist = []

    nconns = 0
    nconns_accepted = 0
    # rejected by id filter
    id_rej = 0
    # rejected by connection filter
    cf_rej = 0
    # only one way seen
    one_ways = 0

    nread = 0
    ntrans = 0
    servtrans_bad = 0
    clitrans_bad = 0


    for file in openfilelist:

	if len(openfilelist) > 1:
	    print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	    print "File %s - %d records" % (file.fnm, file.counters.nrecords)
	    print
	    sys.stdout.flush()
	while 1:
	    if nrecs and nread > nrecs:
		break
	    nread += 1
	    rectype = file.next_type_rec(REC_TCP_HTTP)
	    if rectype == -1: #EOF
		break

	    elif rectype & OPEN_BIT: # notification of TCP open
		file.advance()
		continue

	    elif rectype & HDRS_BIT: # a block of hdrs
		file.advance()
		continue

	    elif rectype == REC_TCP_HTTP: #TCP/HTTP
		#read the lot
		indx, connrec, translist =  np_http_util.get_http_rec(file)

	    else: # any other TCP close record
		# get tconn record and its data
		connrec = nprobe.tcp_conn()
		connrec.get_conn(file)
		print 'X?X?X?'
		file.advance()
		continue

	    # Have now got an HTTP close record - is it one we want?

	    # get the conn_id
	    conn_id = connrec.get_conn_id()
	    flow = connrec.flow_inner
	    nconns += 1
	    if oneconn != None:
		if not oneconn.count(conn_id):
		    id_rej += 1
		    continue
	    elif id < startconn:
		continue
	    elif not nprobe.accept_conn(flow, fspec):
		cf_rej += 1
		continue
	    elif not tfilt.pss(connrec, translist):
                #evlist.append((ull2l(connrec.open()) + ul2l(connrec.clisyn()), EV_RTMTS))
		continue

	    # only want where both directions seen
	    if not (connrec.server_seen() and connrec.client_seen()):
		one_ways += 1
		continue

            if not connrec.no_rtmts_or_gaps():
                evlist.append((ull2l(connrec.open()), EV_RTMTS))
                #continue

	    # 
	    # Now do what we're here for
	    #
	    nconns_accepted += 1
	    open_us = ull2l(connrec.open())
	    clisyn_us = ul2l(connrec.clisyn()) + open_us
	    clisynack_us =  ul2l(connrec.cliacksyn()) + open_us
	    close_us = ull2l(connrec.close())
	    evlist.append((clisyn_us, EV_CONN_SOUGHT))
	    evlist.append((clisynack_us, EV_CONN_RESP))
	    evlist.append((close_us, EV_CONN_CLOSE))
	    for t in translist:
		ntrans += 1
		if t.http_serv_nogood():
		    servtrans_bad += 1
		    break
		if t.http_cli_nogood():
		    clitrans_bad += 1
		    break
		reqstart_us = ul2l(t.http_reqstart_us()) + open_us
		repstart_us = ul2l(t.http_repstart_us()) + open_us
		repend_us = ul2l(t.http_repend_us()) + open_us
		evlist.append((reqstart_us, EV_OB_REQ))
		evlist.append((repstart_us, EV_OB_START))
		evlist.append((repend_us, EV_OB_SERVED))

            if connrec.get_cflags() & TSP_DUP_SYN:
                #print 'dup SYN'
                evlist.append((clisyn_us, EV_DUP_SYN))
		    
		

    print 'Records read %d' % (nread)
    print 'Total HTTP connections %d' % (nconns)
    print 'Total HTTP connections accepted  %d' % (nconns_accepted)
    print '\t%d rejected by flow id filter' % (id_rej)
    print '\t%d rejected by flow filter' % (cf_rej)
    print '\tRejected by tfilter:'
    tfilt.report()
    print
    print '%d Transactions' % (ntrans)
    print '\t%d client invalid %d server invalid' % (clitrans_bad, servtrans_bad)

    buckets = [0, 0, 0, 0, 0, 0, 0, 0]
    aggs =  [[], [], [], [], [], [], [], [], [], []]

    # calculation granularity in us
    #TIME_GRAN = 100000 # = 100ms
    TIME_GRAN = per*1000000

    # time stamp precision
    TIME_PREC = 1000000.0 # = 1us

    # rate factor
    if per_sec:
	RATE_FACT = TIME_PREC/TIME_GRAN
    else:
	RATE_FACT = 1
    
    evlist.sort()
    tbase = evlist[0][0]
    tm = TIME_GRAN
    last_tm = 0
    dlen = 1
    for e in evlist:
	us = long(e[0] - tbase)
	ev = e[1]
	#print 'tm %.6f ev %d' % (us/1000000.0, e[1])
	#print tm,
	#print us
	if us > tm:
            # end of aggregation period
	    while last_tm < tm:
                # zero any missing intervals
		for agg in range(len(buckets)):
		    aggs[agg].append([last_tm/TIME_PREC, 0, agg, []])
                aggs[P_DS].append([tm/TIME_PREC, 0.0, P_DS, []])
                aggs[P_RTMT].append([tm/TIME_PREC, 0.0, P_RTMT, []])
		last_tm = last_tm + TIME_GRAN
                
	    last_tm = last_tm + TIME_GRAN

            # save values for this period
            if buckets[EV_CONN_SOUGHT]:
                p_ds = float(buckets[EV_DUP_SYN])/buckets[EV_CONN_SOUGHT]
                p_rtmt = float(buckets[EV_RTMTS])/buckets[EV_CONN_SOUGHT]
            else:
                p_ds = 0.0
                p_rtmt = 0.0
            aggs[P_DS].append([tm/TIME_PREC, p_ds, P_DS, []])
            aggs[P_RTMT].append([tm/TIME_PREC, p_rtmt, P_RTMT, []])
	    for agg in range(len(buckets)):
		aggs[agg].append([tm/TIME_PREC, buckets[agg]*RATE_FACT, 
				  agg, []])
		buckets[agg] = 0

	    tm = ((us/TIME_GRAN)+1)*TIME_GRAN

        # record this event
	buckets[ev] += 1


    while last_tm < tm:
        aggs[P_DS].append([tm/TIME_PREC, 0.0, P_DS, []])
        aggs[P_RTMT].append([tm/TIME_PREC, 0.0, P_RTMT, []])
	for agg in range(len(buckets)):
	    aggs[agg].append([last_tm/TIME_PREC, 0, agg, []])
	last_tm = last_tm + TIME_GRAN

        
    aggs[P_DS].append([tm/TIME_PREC, 0.0, P_DS, []])
    aggs[P_RTMT].append([tm/TIME_PREC, 0.0, P_RTMT, []])
    for agg in range(len(buckets)):
	aggs[agg].append([tm/TIME_PREC, buckets[agg]*RATE_FACT, agg, []])
        

##     # calculate proportion of conns per period with dup SYNs or rtmts
##     p_ds = []
##     p_rtmt = []

##     for i in range(len(aggs[EV_CONN_SOUGHT])):
        
    
    sets = []
    sets.append(DataSet(aggs[EV_CONN_SOUGHT], DATA_TS,
                        'Connection attempts', 0, None))
 ##    sets.append(DataSet(aggs[EV_CONN_RESP], DATA_TS,
##                         'Connections made', 1, None))
##     sets.append(DataSet(aggs[EV_OB_REQ], DATA_TS,
##                         'Requests', 2, None))
##     sets.append(DataSet(aggs[EV_OB_START], DATA_TS,
##                         'Responses', 3, None))
##     sets.append(DataSet(aggs[EV_OB_SERVED], DATA_TS,
##                         'Responses completed', 4, None))
##     sets.append(DataSet(aggs[EV_CONN_CLOSE], DATA_TS, 'Connections closed', 5, None))
    sets.append(DataSet(aggs[EV_DUP_SYN], DATA_TS, 'Duplicate SYNs', 6, None))
    sets.append(DataSet(aggs[EV_RTMTS], DATA_TS, 'Data RTMTs', 7, None))
    sets.append(DataSet(aggs[P_DS], DATA_TS, 'P Duplicate SYNs', 8, None))
    sets.append(DataSet(aggs[P_RTMT], DATA_TS, 'P Data RTMTs', 9, None))

    if per_sec:
	rf_str = 's'
    elif TIME_GRAN < 1000:
	rf_str = '%dus' % (TIME_GRAN)
    elif 1000 <= TIME_GRAN <= 1000000:
	rf_str = '%dms' % (TIME_GRAN/1000)
    else:
	rf_str = '%ds' % (TIME_GRAN/1000000)

    ylabstr = 'No/%s' % (rf_str)

    try:
	np_Plot(sets, standalone='yes', path=basepath,  
	    title='Server load', xlab='time s', ylab=ylabstr)
    except EmptyDataSetError:
	print '%s - empty data set' % (ds.path)
	sys.exit(1)

	    

##############################################################################


# Call main when run as script
if __name__ == '__main__':
        main()