connsum.py

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

#! /usr/bin/env python
###############################################################################
#                                                                             #
#   Copyright 2005 University of Cambridge Computer Laboratory.               #
#                                                                             #
#   This file is part of Nprobe.                                              #
#                                                                             #
#   Nprobe is free software; you can redistribute it and/or modify            #
#   it under the terms of the GNU General Public License as published by      #
#   the Free Software Foundation; either version 2 of the License, or         #
#   (at your option) any later version.                                       #
#                                                                             #
#   Nprobe is distributed in the hope that it will be useful,                 #
#   but WITHOUT ANY WARRANTY; without even the implied warranty of            #
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the             #
#   GNU General Public License for more details.                              #
#                                                                             #
#   You should have received a copy of the GNU General Public License         #
#   along with Nprobe; if not, write to the Free Software                     #
#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA #
#                                                                             #
###############################################################################


from sys import argv
import getopt
import os
import sys
import re

from np_TCPConn import D_DUPCSYN, D_DUPFREQ, D_CRETRANS, D_NOT_CONNECTED, \
     D_REQNOTSEEN, D_REPNOTSEEN, D_SLOSS, D_BIGDEL

from np_TCPConn import D_BIGDEL

replist = []

def rep(s):
    replist.append(s)

def rep_per(args):
    boss_v, boss_t = args[0]
    if not boss_v:
        return

    rep('%d %s:' % (boss_v, boss_t))
    for v, t in args[1:]:
        if v:
            pcf = 100.0/boss_v
            rep('\t%d %s (%.2f%%)' % (v, t, v*pcf))

def rep_rep():
    try:
        sf = open(sumfilenm, 'w')
    except IOError, s:
        print 'Couldn\'t open summary file', s
        sys.exit(1)
        
    for s in replist:
        print s
        sf.write(s + '\n')

accum_fields = ['totconns',
                'dupcsyn',
                'dupfreq',
                'cretrans',
                'cretrans_p',
                'cretrans_np',
                'not_c',
                'noreq',
                'norep',
                'sloss',
                'sloss_p',
                'sloss_np',
                'dconn',
                'bdconn',
                'nob',
                'nobp',
                'persist',
                'cdpkts',
                'cdpkts_p',
                'crtmts',
                'sdpkts',
                'sdpkts_p',
                'srtmts',
                'nfreq',
                'ncsyn']

class accum:

    def __init__(self, what):
        self.what = what
        for f in accum_fields:
            setattr(self, f, 0)

    def report(self):

        rep(self.what)
        tc = self.totconns
        rep_per([
            (tc, 'Connections'),
            (self.dconn, 'Delayed'),
            (self.bdconn, 'Long delayed'),
            (self.persist, 'Persistent'),
            (self.dupcsyn, 'dupcsyn'),
            (self.ncsyn, 'retransmitted syns'),
            (self.dupfreq, 'dupfreq'),
            (self.nfreq, 'retransmitted firstrequests'),
            (self.not_c, 'not connected'),
            (self.noreq, 'no request'),
            (self.norep, 'no reply'),
            (self.cretrans, 'cretrans (req > 1)'),
            (self.sloss, 'sretrans (req > 1)')
            ])
        rep_per([
            (self.persist, 'Persistent connections'),
            (self.cretrans_p, 'cretrans (req > 1)'),
            (self.sloss_p, 'sretrans (req > 1)')
            ])
            
        rep_per([
            (self.nob, 'Objects'),
            (self.nobp, 'on persistent connections')
            ])

        rep_per([
            (acc.cdpkts, 'Client data pkts'),
            (acc.cdpkts_p, 'on persistent connections'),
            (acc.crtmts, 'retransmitted packets')
            ])

        rep_per([
            (acc.sdpkts, 'Server data pkts'),
            (acc.sdpkts_p, 'on persistent connections'),
            (acc.srtmts, 'retransmitted packets')
            ])

def accum_tot(a, b, c):
    
    for f in accum_fields:
        setattr(c, f, getattr(a, f) + getattr(b, f))

scriptname = os.path.basename(argv[0])
ofnm = None

try:
    optlist, args = getopt.getopt(sys.argv[1:], 'o:')

except getopt.error, s:
    print '%s: %s' % (scriptname, s)
    usage(scriptname)
    sys.exit(1)

for opt in optlist:
    if opt[0] == '-o':
        ofnm = opt[1]

start_re = re.compile('# Run start = ([0-9]*).*')
start = 0

accums = [accum('NOT IN TREES:'), accum('IN_TREES:'), accum('TOTAL:')]
 

basedir = os.path.dirname(args[0])

frange = []
suffs = []
pref_re = re.compile('(.\.rep\.\d*)-(.\.rep\.\d*)(\..*)')
print 'files from', os.getcwd(), ':'
rep('files from %s:' % (os.getcwd()))
for fnm in args:
    fnm = os.path.basename(fnm)
    print '   ', os.path.basename(fnm)
    m = pref_re.match(fnm)
    if m:
        #print m.group(1), m.group(2), m.group(3)
        frange.append(m.group(1))
        frange.append(m.group(2))
        suffs.append(m.group(3))

if frange:
    print frange
    for suff in suffs[1:]:
        if suff != suffs[0]:
            print 'Ouch mixed suffix:', suff
    frange.sort()
    ofnm = frange[0] + '-' + frange[-1] + suffs[0]
    #print range

else:
   pref_re = re.compile('(.\.rep\.\d*)\.(.*)') 
   m = pref_re.match(os.path.basename(fnm))
   if m: 
        #print m.group(1), m.group(2)
        ofnm = m.group(0)

if not ofnm:
    ofnm = raw_input('output files base? - base is %s\n?' % (basedir))
    if not ofnm:
        ofnm = os.path.basename(fnm)
outfbase = os.path.join(basedir, ofnm)

sumfilenm = outfbase + '.Summary'
print 'Summary file is', sumfilenm
rep('Summary file is %s' % (sumfilenm))



nconn = 0
badl = 0
for fnm in args:
    print os.path.basename(fnm)
    f = open(fnm, 'r')
    lno = 0
    for l in f.readlines():
        l = l.replace('\n', '')
        if l[0] == '#':
            m = start_re.match(l)
            if m:
                strt = long(m.group(1))
                print 'start = ', start
                if start == 0:
                    start = strt
                strt -= start
            continue
        sf = l.rstrip().split(' ')
        lsf = len(sf)
        if lsf < 8:
            print 'connsum - bad line %s %d: %s %d fields' % (fnm, lno, l, lsf)
            badl += 1
            continue
        nconn += 1
        id = sf[0]
        pageno = int(sf[1])
        tm = float(sf[2]) - strt
        _ncdpkts = int(sf[3])
        ncdpkts = int(sf[4])
        nsdpkts = int(sf[5])
        intree = int(sf[6])
        persist = int(sf[7])

        if lsf > 8:
            dly = sf[8:]
            for i in range(len(dly)):
                dly[i] = int(dly[i])
            dflags = dly[-1]
        else:
            dflags = 0
            
        acc = accums[intree]
        acc.totconns += 1
        if persist:
            if persist == 1:
                print 'persist goof'
                sys.exit (1)
            acc.nob += persist
            acc.nobp += persist
            acc.persist += 1
            acc.cdpkts_p += ncdpkts
            acc.sdpkts_p += nsdpkts
        else:
            acc.nob += 1

        acc.cdpkts += ncdpkts
        acc.sdpkts += nsdpkts
        acc.ncsyn += 1 # must have seen one
        if dflags:
            acc.dconn += 1
            cri = 0
            if dflags & D_DUPFREQ:
                acc.dupfreq += 1
                acc.nfreq += dly[cri]
                cri += 2
            elif dflags & D_REQNOTSEEN and not (dflags & D_NOT_CONNECTED):
                acc.noreq += 1
                cri += 2
            else:
                acc.nfreq += 1
            if dflags & D_CRETRANS:
                acc.cretrans += 1
                if persist:
                    acc.cretrans_p += 1
                else:
                    acc.cretrans_np += 1
                acc.crtmts += dly[cri]
                cri += 2
            if dflags & D_DUPCSYN:
                acc.dupcsyn += 1
                acc.ncsyn += dly[cri]
                cri += 2
            if dflags & D_NOT_CONNECTED:
                acc.not_c += 1
            if dflags & D_REPNOTSEEN and not (dflags & D_REQNOTSEEN):
                acc.norep += 1
            if dflags & D_SLOSS:
                acc.sloss += 1
                if persist:
                    acc.sloss_p += 1
                else:
                    acc.sloss_np += 1
                acc.srtmts += (dly[-2] + dly[-3])
            if dflags  & D_BIGDEL:
                acc.bdconn += 1

if not nconn:
    print 'No conns in Connections file(s)'
    sys.exit(1)

accum_tot(accums[0], accums[1], accums[2])
print nconn, 'conns'
for i in [1, 0, 2]:
    acc = accums[i]
    acc.report()

print badl, 'bad lines'
rep_rep()