histo.py

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

#! /usr/bin/env python
###############################################################################
#                                                                             #
#   Copyright 2005 University of Cambridge Computer Laboratory.               #
#                                                                             #
#   This file is part of Nprobe.                                              #
#                                                                             #
#   Nprobe is free software; you can redistribute it and/or modify            #
#   it under the terms of the GNU General Public License as published by      #
#   the Free Software Foundation; either version 2 of the License, or         #
#   (at your option) any later version.                                       #
#                                                                             #
#   Nprobe is distributed in the hope that it will be useful,                 #
#   but WITHOUT ANY WARRANTY; without even the implied warranty of            #
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the             #
#   GNU General Public License for more details.                              #
#                                                                             #
#   You should have received a copy of the GNU General Public License         #
#   along with Nprobe; if not, write to the Free Software                     #
#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA #
#                                                                             #
###############################################################################


from array import array
from math import ceil
import sys
import re
from types import IntType, FloatType, TupleType, ListType, StringType
from math import sqrt

###############################################################################
###############################################################################

NBUCKETS_DEF = 100

###############################################################################
###############################################################################

class HistogramError:

    def __init__(self, val):
        self.val = val

    def __str__(self):
        return 'HistogramError: ' + self.val

###############################################################################
###############################################################################

class HistogramHighRangeError:

    def __init__(self, val):
        self.val = val

    def __str__(self):
        return 'Histogram Above Range Error: ' + self.val

###############################################################################
###############################################################################

class HistogramLowRangeError:

    def __init__(self, val):
        self.val = val

    def __str__(self):
        return 'Histogram Below Range Error: ' + self.val

###############################################################################
###############################################################################


class Histogram:

    """Class Histogram - creates and populates histogram.

       Instantiation:
       
         Instantiated with the following optional keyword arguments
           (default value shown):-
           'lower=None' - Numeric value sets histogram lower bound
           'upper=None' - Numeric value sets histogram lower bound
           'bucketsz=None' - Numeric value sets binning width
           'nbuckets=None' - Numeric value sets number of bins
           'ntrigvals=None' - Autorange trigger value
           'save_orange=0' - 1 Sets save out of range values
           'rangeexcept=0' - 1 Raise out of range exceptions
           'verbose=0' - 1 enables blather on about what it's doing

       Public methods:

         The following three methods are intended to be public:

           add(value)
             add value to the histogram (see below for out of bounds handling)

             values may be integer/float or may be a list/tuple of a value/key
             pair

           extend(value)
             extend upper/lower bound to allow incorporation of value (which
               may be a value/key pair). By default value is incorporated, but
               will not be if the optional argument 'addval=0' is used

           results()
             return a dictionary containing the histogram state on completion
               - takes the following optional keyword arguments
                 (default values shown):-
               'printit=0' - 1 gives pretty print out of results
               'draw=0' - 1 fires up np_plot of histogram
               'file=''' - write results to a valid file name or path
               'title=''' - give the results this title
               'comment=''' - append this comment to the results

             the dictionary contains the following fields:
             'title' - title
             'comment' - comment
             'lower' - lower bound
             'upper' - upper bound
             'bucket size' - bin width
             'nbuckets' - number of bins
             'nsamples' - number of samples presented
             'min' - minimum sample value incorporated
             'max' - maximum ditto
             'samplemin' - minimum sample value presented
             'samplemax' - maximum ditto
             'nlow' - number of below bound values presented (not incorporated)
             'nhigh' - above bound ditto 
             'lows' - list of below bound values (or value/key pairs) not
                incorporated - only if instantiated with 'orange=1')
             'highs' - ditto above bound values
             'low_range_exceptions' - number of exceptions thrown by below
                bound values (only if instantiated with 'rangeexcept=0')
             'high_range_exceptions' - ditto by above bound values
             'values' - list of bin values and sample counts
               N.B. Bin values are the *upper* bin pivot, bin counts are count
                 of samples <= upper pivot
             'mean' - mean of incorporated samples
             'sample_mean' - mean of all samples presented
             'sd' - S.D. of incorporated samples
             'sample_sd' - ditto of all samples presented
             'modes' - list of modes of samples incorporated

             Optional arguments will specify the behaviour desired when
               calling results(), and the dictionary fields can be used for
               closer examination (e.g., to check for out of bounds values,
               retrieve mean, etc.)

       Operation and Auto-ranging:
      
         If lower and upper histogram bounds are specified all samples are
           immediately added in, and out of range samples dealt with as
           determined by the 'save_orange' and 'rangeexcept' arguments.
         If neither, or only one bound, are specified the sample values are
           stored until a trigger number of samples - specified using
           the 'ntrigvalues' flag - have been presented, at which point an
           upper/lower bound, or both, are automatically calculated based upon
           the current max/min stored sample values, and the stored values
           are incorporated.
           Bucket widths or nbuckets are calculated based upon the specified
           nbuckets or width - if neither are specified the default
           nbuckets = 100 is used.
         The default trigger value of 0 results in all samples being stored and
         auto range calculations being performed on the entire sample set at
         completion (call to results()).

       Exceptions:

         The following exceptions may be raised:

           HistogramError - during instantiation due to incorrect arguments
                          - on calls to extend() with in-bounds argument
                          - on calls to results() with no values added

           HistogramLowRangeError
           HistogramHighRangeError - as a result to calls to add() with out of
             bounds values - if enabled
             
         the value of all exception objects is an explanatory string

       Out of bounds handling:

         Out of bounds values are handled according to the values of the 
            'save_orange' and 'rangeexcept' instantiation arguments.
            If neither are set the default is to simply count the number of
            samples presented and not incorporated.

            OOB samples can be identified by setting either of the above
            arguments, exceptions allowing such samples to be immediately
            identified without incurring the memory penalty of accumulating
            a list of values when presenting large data sets. If value/key pairs
            are used as the argument to add() and 'save_orange' is set then the
            keys saved in the 'highs' and 'lows' fields returned by results()
            can be used to identify the origin, as well as the value, of OOB
            samples.

       Examples:

         Examples of use of the class can be found in main() of histo.py and the
         main() function of np_TCPGet.py
         
       """

    def __init__(self, lower=None, upper=None, bucketsz=None, nbuckets=None, ntrigvals=None, save_orange=0, rangeexcept=0, verbose=0):

        self.verbose = verbose

        for t in [nbuckets, ntrigvals]:
            if t and not type(t) is IntType:
                raise HistogramError('if specified nbuckets and ntrigvals must be integers')

        self.floating = self.floatvals = 0
        for t in [lower, upper, bucketsz]:
            if type(t) is FloatType:
                self.floating = 1
            elif t and not type(t) is IntType:
                raise HistogramError('if specified lower, upper and bucketsz must be integers or floats')

        if lower != None and upper != None:
            if lower > upper:
                raise HistogramError('specified lower bound > upper bound')
            if lower == upper:
                raise HistogramError('specified upper and lower bounds are equal')
            if bucketsz != None and bucketsz >= upper - lower:
                raise HistogramError('specified bucketsz >= range')
        
        self.lower = lower
        self.upper = upper

        if bucketsz != None:
            if not (bucketsz > 0):
                raise HistogramError('specified bucketsz <= 0')
        elif nbuckets == None:
            nbuckets = NBUCKETS_DEF
        self.bucketsz = bucketsz

        if nbuckets != None and nbuckets <= 0:
            raise HistogramError('specified nbuckets <= 0')
        self.nbuckets = nbuckets

        if nbuckets and bucketsz:
            raise HistogramError('both nbuckets and bucketsz specified')

        if ntrigvals != None:
            if ntrigvals < 0:
                raise HistogramError('specified ntrigvals < 0')
        self.ntrigvals = ntrigvals

        self.rangeexcept = rangeexcept

        if rangeexcept and save_orange:
            raise HistogramError('save out of bounds values specified and exceptions enabled')

        self.saved = []
        self.extends = []
        self.ha = None

        self.orange = save_orange
        self.lows = []
        self.highs = []
        self.nlows = self.nhighs = 0
        self.nlre = self.nhre = 0

        self.max = self.min = None
        self.samplemax = self.samplemin = None
        self.nsamples = 0 # total samples presented
        self.N = 0 # samples added
        self.u = self.s2 = None # initialised at first sample
        self.sampleu = self.samples2 = None # initialised at first sample

        if lower != None and upper!= None:
            if ntrigvals != None:
                raise HistogramError('range specified together with ntrigvals')
            self.makearray()

###############################################################################

    def makearray(self):

        self.saved.sort()

        autorange = 0

        if self.lower != None:
            self.extends.append(self.lower)
            lower = min(self.extends)
            if lower != self.lower:
                autorange = 1
        else:
            l = self.saved[0]
            t = type(l)
            if t is TupleType or t is ListType:
                l = l[0]
            lower = l
            autorange = 1

        self.lower = lower
            
        if self.upper != None:
            self.extends.append(self.upper)