np_rets.py

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

#! /usr/bin/env python
###############################################################################
#                                                                             #
#   Copyright 2005 University of Cambridge Computer Laboratory.               #
#                                                                             #
#   This file is part of Nprobe.                                              #
#                                                                             #
#   Nprobe is free software; you can redistribute it and/or modify            #
#   it under the terms of the GNU General Public License as published by      #
#   the Free Software Foundation; either version 2 of the License, or         #
#   (at your option) any later version.                                       #
#                                                                             #
#   Nprobe is distributed in the hope that it will be useful,                 #
#   but WITHOUT ANY WARRANTY; without even the implied warranty of            #
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the             #
#   GNU General Public License for more details.                              #
#                                                                             #
#   You should have received a copy of the GNU General Public License         #
#   along with Nprobe; if not, write to the Free Software                     #
#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA #
#                                                                             #
###############################################################################


##############################################################################
## 
##  np_rets.py
## 
##  Inputs series of Nprobe logs, reads in HTTP transactions, counts 
##  frequency of various server return codes, object sizes, etc
## 
############################################################################ 

from string import center
import glob
import os
from os.path import split
import sys
from types import StringType
from sys import argv
import getopt
import re

import Numeric

from nprobe import REC_TCP_HTTP, TRANS_FINISHED, TRANS_INCOMPLETE, \
TCP_SERV_FIN, TCP_FULL_CLOSE, TCP_EFFECTIVE_CLOSE, TCP_QUICK_CLOSE, \
http_server_returncode_string, http_server_objtype_string, accept_conn, filter_help, HTTP_METHOD_GET, http_client_method_string, MAX_NTRANS

from np_file_util import get_files
from np_http_util import allocate_http_reusable_objects, \
     get_http_rec_and_trans, get_http_rec
#from np_http_util import *

from np_plot import np_Plot, DataSet, DATA_HIST, DATA_PDF, DATA_CDF, MODE_HIST, MODE_PDF, MODE_CDF, STYLE_LINES, STYLE_BARS
from minmax import MIN, MAX

from np_WebAgents import WebAgents
from np_FileTypes import FileTypes

from  np_longutil import ull2l

############################################################################

MAX_RETCODE = 600
MAX_METHOD = 256

OBSZ_MIN = 0
OBSZ_MAX = 1000*100
OBSZ_BIN = 50

NBINS = ((OBSZ_MAX-OBSZ_MIN)/OBSZ_BIN) + 1

# Threshold for individula object reporting
VERY_LARGE_OB_THRESH = 10*OBSZ_MAX

BIGNUMBER = 100000000

############################################################################

def usage(scriptname):

    print
    print scriptname + ':', 'Inputs series of Nprobe logs, reads in HTTP transactions, counts\n  frequency of various server return codes, object sizes, etc.\n  Places results in repfile directory np_rets subdir.'
    
    print "usage: " + scriptname + " [flag(s)] <rep-file-list | rep directory>"
    print 'Flags:\n'
    print '\t-F Apply simple filter to input (use -Fh for details)'
    print '\t-s<spec> Show object size distributions\n\t\tr - as histogram\n\t\tp pdf\n\t\tc cdf'
    print '\t-a Save list of all User Agents and Servers encountered'
    print '\t-t<dir> Check HTTP claimed object types against (partial or complete)\n\t  saved objects using file magic.'
    print '\t-h This help.'

    sys.exit(1)

#############################################################################

def openf(fnm, mode):

    try:
        f = open(fnm, mode)
    except IOError, s:
        print 'Error'
        print str(s)
        sys.exit(1)

    return f

############################################################################# 

def sort_by_occurences_rev(a, b):

    return b[1] - a[1]

#############################################################################

def record(f, s):

    print s
    f.write(s + '\n')

#############################################################################

def write_file(f, s):

    f.write(s + '\n')

#############################################################################

def object_type_string(otype):
    
    return http_server_objtype_string(otype).replace('/', '-')

#############################################################################

def get_mode(flag):

    if flag == 'r':
        return (DATA_HIST, MODE_HIST, STYLE_BARS, '.hist')
    elif flag == 'p':
        return(DATA_PDF, MODE_PDF, STYLE_BARS, '.pdf')
    elif flag == 'c':
        return(DATA_CDF, MODE_CDF, STYLE_LINES, '.cdf')
    else:
        print 'Unrecognised drawing mode'
        sys.exit(1)

#############################################################################

def do_methods(filepath, methods, ntrans):

    filepath = filepath + 'request_methods'
    f = open(filepath, 'w')

    record(f, "\nHTTP request methods (total %d valid objects)\n" \
           % (ntrans))

    sortlist = []
    for i in range(MAX_METHOD):
        n = methods[i]
        if n:
            sortlist.append((n, i))

    sortlist.sort()
    sortlist.reverse()

    record(f, "%s%s%s\n\n" % (center("Method", 15), center("Total", 15),
                           center("% of Whole", 12)))

    for n, meth in sortlist:
        record(f, "%15s%15d%12.3f" % (http_client_method_string(meth).ljust(15), n, (n*100.00)/ntrans))

    record(f, '\n')
    sumfile.write('%d HTTP request methods seen - see %s\n' % (len(sortlist),
                                                               filepath))
    record(sumfile, '\n================================================================\n\n')

    f.close
        

    


#############################################################################

def do_retcodes(filepath, sumfile, vals, totrans):

    for v, label in vals:
        ntrans = v[0]
        rvals = v[4]

        filepath = filepath + label + '.retcodes'
        f = open(filepath, 'w')

        rlist = []

        f.write('HTTP server return codes for %s - by code (%d transactions %.3f%% of total transactions)\n\n' \
               % (label, ntrans, (ntrans*100.00)/totrans))

        f.write("%s%s%s\n\n" % (center("Code", 5), center("Text", 30),
                               center("% Total", 12)))

        for i in range(MAX_RETCODE):
            if rvals[i] > 0:
                f.write("%5d%30s%12.3f\n" % (i, http_server_returncode_string(i),
                                             (rvals[i]*100.0)/totrans))
                rlist.append((rvals[i], i))


        f.write("\n\n\nHTTP server return codes for %s - by frequency\n\n" % (label))

        f.write("%s%s%s\n\n" % (center("Code", 5), center("Text", 30),
                                  center("% Total", 12)))

        rlist.sort()
        rlist.reverse()

        for n, code in rlist:
            f.write("%5d%30s%12.3f\n" % (code,
               http_server_returncode_string(code), (n*100.0)/totrans))


        f.close()
        
    record(sumfile, 'For analysis of server return codes  - see %s\n' % (filepath))
    record(sumfile, '\n================================================================\n\n')


#############################################################################


def do_obtypes(filepath, sumfile, typelist, filetypes, ntrans):

    filepath = filepath + 'object_types'
    f = open(filepath, 'w')

    hdstr = "\nHTTP object types with return code 200 (total %d valid objects)\n" % (ntrans)
    record(sumfile, hdstr)
    f.write(hdstr + '\n')

    record(sumfile, "%s%s%s\n\n" \
           % (center("Type", 30), center("Total", 15),
              center("% of Whole", 16)))

    f.write("%s%s%s%s%s%s\n\n\n" \
           % (center("Type", 30), center("Total", 15),
              center("% of Whole", 16), center('Unfinished', 18),
              center('Non-200 responses', 18), center('Min/Max (Finished)', 25)))

    for fins, unfins, type in typelist:
        nfins = fins[0]
        nunfins = unfins[0]
        tot = nfins + nunfins
        non_200 = fins[3] + unfins[3]

        maxsz = max(fins[6], unfins[6])
        if fins[0]:
            minstr = '%d' % fins[5]
        else:
            minstr = '-'
            
        f.write("%-30s%15d%12.3f%10d (%7.3f%%) %10d (%7.3f%%) %8s %10d\n" % \
               (type, tot, (tot*100.00)/ntrans, nunfins, (nunfins*100.00)/tot, non_200,
                (non_200*100.00)/tot, minstr, fins[6]))
            
        record(sumfile, "%-30s%15d%12.3f" % \
               (type, tot, (tot*100.00)/ntrans))

    record(f, '\n')
    if filetypes:
        filetypes.report_diffs(file=f)
    record(sumfile, '\nFor greater detail see %s\n' % (filepath))
    record(sumfile, '\n================================================================\n\n')

    f.close


#############################################################################
    

def do_obsz(filepath, sumfile, vals, totrans, non_200, modeflag, showflag):
        
    szdir = filepath + 'ob_size_dists'
    try:
        os.mkdir(szdir)
        #print 'Created sizes directory %s' % (szdir)
    except OSError, s:
        if str(s).find('exists'):
            pass
            #print 'Results directory %s already exists' % (szdir)
        else:
            print str(s)
            sys.exit(1)
        
    dtype, mode, style, fsuff = get_mode(modeflag)

    showsets = []
    cycle = 0

    for fin, unfin, type in vals:

        nfin = fin[0]
        nfin_notok = fin[3]
        nfinok = nfin - nfin_notok
        
        if nfinok == 0:
            continue
        
        nunfin = unfin[0]
        nunfin_notok = unfin[3]
        nunfinok = nunfin - nunfin_notok

        fp = szdir + '/' + type
        hf = open(fp + '.hist', 'w')
        pf = open(fp + '.pdf', 'w')
        cf = open(fp + '.cdf', 'w')

        data = fin
        bigs = data[2]  
        bigs.sort()
        minsz = data[5]
        maxsz = data[6]
        osz = data[1]
        zerolens = osz[0]

        typetot = nfin + nunfin
        nt = totrans - nunfin - non_200

        for f in [hf, pf, cf]:

            write_file(f, '# Object size distribution for HTTP %s objects\n#\t%d complete objects sized %d - %d bytes\n#\t(%.3f%% of all completed ok transactions, %.3f%% of all transactions)'\
                   % (type, nfinok, minsz, maxsz, (100.0*nfinok)/nt, (100.0*nfinok)/totrans))
            write_file(f, '\n# %d Total transactions of this type' % (typetot))
            write_file(f, '# Of these:\n')
            totok = nfinok + nunfinok
            write_file(f, '#\t%d Retcode 200 - ok (%.3f%%)' % (totok, (totok*100.0)/typetot))
            totnotok = nfin_notok + nunfin_notok
            write_file(f, '#\t%d Not retcode 200 (%.3f%%)' % (totnotok, (totnotok*100.0)/typetot))
            write_file(f, '#\t%d Finished ok (%.3f%%)' % (nfinok, (nfinok*100.0)/typetot))
            write_file(f, '#\t%d Finished not ok (%.3f%%)' % (nfin_notok, (nfin_notok*100.0)/typetot))
            write_file(f, '#\t%d Unfinished ok (%.3f%%)' % (nunfinok, (nunfinok*100.0)/typetot))
            write_file(f, '#\t%d Unfinished not ok (%.3f%%)\n' % (nunfin_notok, (nunfin_notok*100.0)/typetot))


            write_file(f, '\n# %d Zero length completed ok (%.3f%% of completed ok)' % (zerolens, (zerolens*100.00)/nfinok))
            write_file(f, '\n\n')

        bin = 0
        pts = []
        accum = 0.0


        for i in range(NBINS):
            n = osz[i]
            if n:
                p = float(n)/nfinok
                accum += p
                write_file(hf, '%15d %15d' % (bin, n))
                write_file(pf, '%15d %7.6f' % (bin, p))
                write_file(cf, '%15d %7.6f' % (bin, accum))
                if dtype == DATA_PDF:
                    v = p
                elif dtype == DATA_CDF:
                    v = accum
                else:
                    v = n
                pts.append([bin, v, cycle])
            bin += OBSZ_BIN

        if bigs:
            bigs.append(bigs[-1] + OBSZ_BIN)
            while bin < bigs[0]:
                bin += OBSZ_BIN
            n = 0
            for sz in bigs:
                if sz <= bin:
                    n += 1
                    continue
                else:
                    p = float(n)/nfinok
                    accum += p
                    write_file(hf, '%15d %15d' % (bin, n))
                    write_file(pf, '%15d %7.6f' % (bin, p))
                    write_file(cf, '%15d %7.6f' % (bin, accum))
                    if dtype == DATA_PDF:
                        v = p
                    elif dtype == DATA_CDF:
                        v = accum
                    else:
                        v = n
                    pts.append([bin, v, cycle])
                    n = 1
                    while bin < sz:
                        bin += OBSZ_BIN

        
        for f in [hf, pf, cf]:
            f.close() 

        if pts:
            showsets.append(DataSet(pts, dtype, type, cycle, mode))
            cycle += 1

    if showflag:

        totypes = len(showsets)
        ntypes = 0
        nshown = 0
        grogs = []
        prompt = 'first'

        TYPES_AT_ONCE = 5

        for set in showsets:
            grogs.append(set)
            ntypes += 1
            nshown += 1
            if nshown == TYPES_AT_ONCE or ntypes == totypes:
                rep = raw_input('Show size distribution for %s %d types? y/n ' % (prompt, nshown))
                if rep and rep[0] == 'y':
                    np_Plot(grogs, filepath, title='Distribution of HTTP object sizes %s' % (fsuff.replace('.', ' - ')), style=style, mode=mode)
                grogs = []
                nshown = 0
                prompt = 'next'

    record(sumfile, 'Object size distributions written to %s' % (szdir+'/'))
    record(sumfile, '\n================================================================\n\n')

#############################################################################

def do_agents(filepath, sumfile, save_agents, agents):

    f = openf(filepath + 'agents', 'w')
    agents.aggregate_h()
    agents.report_h(file=f)
    if save_agents:
        agents.save_h(filepath)

    record(sumfile, 'Full server and user agent analysis written to %s' % (filepath + 'agents'))
    record(sumfile, '\n================================================================\n\n')

#############################################################################

def do_transconns(filepath, sumfile, transperconn, npers, npers1):
    
    tcdir = filepath + 'transconns'
    
    try:
        os.mkdir(tcdir)
        #print 'Created sizes directory %s' % (szdir)
    except OSError, s:
        if str(s).find('exists'):
            pass
            #print 'Results directory %s already exists' % (szdir)
        else: