np_act.py

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

#! /usr/bin/env python
###############################################################################
#                                                                             #
#   Copyright 2005 University of Cambridge Computer Laboratory.               #
#                                                                             #
#   This file is part of Nprobe.                                              #
#                                                                             #
#   Nprobe is free software; you can redistribute it and/or modify            #
#   it under the terms of the GNU General Public License as published by      #
#   the Free Software Foundation; either version 2 of the License, or         #
#   (at your option) any later version.                                       #
#                                                                             #
#   Nprobe is distributed in the hope that it will be useful,                 #
#   but WITHOUT ANY WARRANTY; without even the implied warranty of            #
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the             #
#   GNU General Public License for more details.                              #
#                                                                             #
#   You should have received a copy of the GNU General Public License         #
#   along with Nprobe; if not, write to the Free Software                     #
#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA #
#                                                                             #
###############################################################################


##############################################################################
## 
##
## Extract probe activity and resource usage from Nprobe logs
## 
## 
############################################################################ 

import string
import os
import sys
import nprobe
from sys import argv
import getopt
from signal import *
import socket

from traceback import print_stack

import np_file_util
import np_http_util
from  np_longutil import *

from np_plot import *

############################################################################

def usage(scriptname):
    print "usage: %s [-h(elp)] <rep-file-list>" % (scriptname)

    sys.exit(1)

#############################################################################

def handle_sigint(n, f):
    print 'SIGINT'
    print_stack()
    sys.exit(0)

#############################################################################

def places(f):
    
    places = 1
    while modf(f)[0]:
        places += 1
        f *= 10
    return places

############################################################################	
	

scriptname = os.path.basename(argv[0])
to_read = None

try:
    optlist, args = getopt.getopt(sys.argv[1:], 'hr:')

except getopt.error, s:
    print '%s: %s' % (scriptname, s)
    usage(scriptname)
    sys.exit(1)

optstr = ''

for opt in optlist:
    if opt[0] == "-h":
	usage(scriptname)
    if opt[0] == "-r":
	to_read = int(opt[1])
        

signal(SIGINT, handle_sigint)

REC_WIRE_PERIOD_REPORT = nprobe.REC_WIRE_PERIOD_REPORT
REC_PERIOD_REPORT = nprobe.REC_PERIOD_REPORT
REC_OTHER = nprobe.REC_OTHER_ALL
REC_PROCSTAT = nprobe.REC_OTHER_PROCSTAT
REC_WRAPPER = nprobe.REC_OTHER_WRAPPER
WRAPPER_TIME_CALLS = nprobe.WRAPPER_TIME_CALLS
WRITE_BLKSZ = nprobe.WRITE_BLKSZ

openfilelist, counters, basepath = np_file_util.get_files(args)

nread = 0

first_ts = 0 # time at start

last_uts = 0 # time of last rusage record
last_utm = 0 # amount of last user rusage
last_stm = 0 # amount of last system rusage

## last_majflt = 0
## last_nswap = 0
## last_blkio = 0
## last_cswitches = 0


last_times_ip_uusage = 0 # amount of last IP call rusage
last_times_ip_susage = 0
last_times_tcp_uusage = 0 # amount of last TCP call rusage
last_times_tcp_susage = 0

last_rts = 0 # time of last period record
last_w_rts = 0 # time of last pkt driven period record
last_times_ts = 0 # time of last call rusage record

last_tous_in = 0
last_tous_out = 0
last_frus_in = 0
last_frus_out = 0

lastrtm = 0

# where to collect the stats
urusage = []
srusage = []
trusage = []
## maxrss = []
## majflt = []
## nswap = []
## blkio = []
## cswitches = []

## times_ip_uusage = []
## times_ip_susage = []
## times_tcp_uusage = []
## times_tcp_susage = []
## times_udp_uusage = []
## times_udp_susage = []
## times_tcp_inseq_uusage = []
## times_tcp_inseq_susage = []
## times_tcp_catchup_uusage = []
## times_tcp_catchup_susage = []
## times_tcp_serv_uusage = []
## times_tcp_serv_susage = []
## times_http_uusage = []
## times_http_susage = []
## times_html_uusage = []
## times_html_susage = []

wire_bw = []
tcp_bw = []
udp_bw = []
http_bw = []
html_parse = []
wire_pkts = []
tcp_pkts = []
udp_pkts = []
http_pkts = []
rwritten = []
dwritten = []
twritten = []
rdumped = []
ddumped = []
tdumped = []
max_fetch_int = []
bheld = []
fails = []
IP_ulen = []

tcp_to = []
udp_to = []
tcp_seq_to = []

tous_in = []
tous_out = []
frus_in = []
frus_out = []

# do but packet arrival driven
w_wire_bw = []
w_tcp_bw = []
w_udp_bw = []
w_http_bw = []
w_html_parse = []
w_wire_pkts = []
w_tcp_pkts = []
w_udp_pkts = []
w_http_pkts = []

# stats from /proc/self
proc_utm = []
proc_stm = []
proc_ttm = []
proc_majflt = []
proc_nswap = []

# stats from /proc/stats
proc_itot = []
proc_csw = []


# running maximum for scaling
## max_maxrss = 0
## max_majflt = 0
## max_nswap = 0
## max_blkio = 0
## max_cswitches = 0

## max_times_ip_uusage = 0
## max_times_ip_susage = 0
## max_times_tcp_uusage = 0
## max_times_tcp_susage = 0
## max_times_udp_uusage = 0
## max_times_udp_susage = 0
## max_times_tcp_inseq_uusage = 0
## max_times_tcp_inseq_susage = 0
## max_times_tcp_catchup_uusage = 0
## max_times_tcp_catchup_susage = 0
## max_times_tcp_serv_uusage = 0
## max_times_tcp_serv_susage = 0
## max_times_http_uusage = 0
## max_times_http_susage = 0
## max_times_html_uusage = 0
## max_times_html_susage = 0

max_wmbs = 0
max_tmbs = 0
max_umbs = 0
max_hmbs = 0 
max_wpps = 0
max_tpps = 0
max_upps = 0
max_hpps = 0
max_hparsed = 0
max_repwritten = 0
max_dumpwritten = 0
max_twritten = 0
max_repdumped = 0
max_dumpdumped = 0
max_tdumped = 0
max_mfi = 0
max_bheld = 0
max_fails = 0

max_IP_ulen = 0

max_tcp_to = 0
max_udp_to = 0
max_tcp_seq_to = 0

max_tous_in = 0
max_tous_out = 0
max_frus_in = 0
max_frus_out = 0

# do but packet arrival driven
w_max_wmbs = 0
w_max_tmbs = 0
w_max_umbs = 0
w_max_hmbs = 0
w_max_wpps = 0
w_max_tpps = 0
w_max_upps = 0
w_max_hpps = 0
w_max_hparsed = 0

# stats from /proc/self
max_proc_utm = 0
max_proc_stm = 0
max_proc_ttm = 0
max_proc_majflt = 0
max_proc_nswap = 0

# stats from /proc/stats
max_proc_itot = 0
max_proc_csw = 0

ru = nprobe.np_rusage()
pr = nprobe.period_report()
wr = nprobe.wrapper_record()
pcr = nprobe.procstat_rec()

file_closes = []

for file in openfilelist:
    
    fnm = os.path.basename(file.fnm)
    hz = file.get_hz()
    ncpus = file.get_ncpus()
    #if len(openfilelist) > 1:
    if 1:
	print "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	print "File %s - %d records" % (fnm, file.counters.nrecords)
	print
	sys.stdout.flush()
        
    while 1:
        
        nread += 1
        if to_read and nread > to_read:
            break
         
        #rectype = file.find_next_period_record()
        rectype = file.next_type_rec(REC_OTHER)
        #print rectype
	if rectype == -1: #EOF
	    break

        elif rectype == REC_WIRE_PERIOD_REPORT:
            pr.read_reprec(file, ru, 0)

            #
            # stats stuff - but driven by packet arrival times
            #
            rts = tv2l(pr.get_ts())
            if first_ts == 0:
                first_ts = rts
            if rts == last_w_rts:
                print 'rusage - duplicated time stamp',
                #print float(rts-first_ts)/1000000
                continue
            per = rts - last_w_rts
            last_w_rts = rts
            ts = float(rts - first_ts)/1000000
            #print ts, per