np_tcpget.py

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

                        except KeyError:
                            continue
                    elif want(connrec):
                        frecs = [(file.indx, REC_TCP_ALL, file, offset)]
                        hlist = []
                    else:
                        continue

                if hdrs and not mmh:
                    get_hdrs(connrec, hlist)

                # create Filerec for connection
                fr = FileRec('TCPConn', frecs)
                
                if opord:
                    if connid == opens[0]:
                        del opens[0]
                        yield mconn()
                        while opens:
                            dent = conn_ids[opens[0]]
                            conn = dent[2]
                            if not conn:
                                break
                            #print opens
                            if mm:
                                conn = conn.reconstruct()
                                conn.adjust_tm_offsets(run_start)
                            yield conn
                            del dent
                            del opens[0]
                    else:
                        if mm:
                            conn_ids[connid][2] = fr
                        else:
                            conn_ids[connid][2] = mconn()
                else:
                    yield mconn()
                    if ops:
                        del opens[0]
                        del conn_ids[connid]


                if not nread%100 and not done:
                    overprint('Record %d' % (nread))

            if done:
                break

        overprint('%d relevant close records' % (nread))

        ectrs.printself(None)
        print
        if opord:
            print '%d pending connections encountered but not delivered awaiting ordering gap' % (len(opens))


############################################################################

    def set_args(self, argv, caller_optstr):

        #
        # Aggregate cl list of connections wanted
        #
        def get_vals(s, attr):
            vals = []
            for f in s.split(','):
                vals.append(int(f))
            setattr(self, attr, vals)
        #
        # Aggregate cl list of hosts wanted
        #
        def get_hosts_wanted(s, attr):
            hosts = []
            for f in s.split(','):
                #print f
                #print '%d' % (string.atoi(f))
                hosts.append(_inet_aton(f))
            setattr(self, attr, hosts)

        #
        # Body of method starts here
        #

        for c in self.optstr:
            if c in caller_optstr and c != ':':
                print 'TCPGet ERROR: \'%s\' is flag for both TCPGet and calling script' % (c)
                sys.exit(1)

        self.caller = argv[0]
        caller_optlist = self.caller_optlist = []

        self.caller = argv[0]

        try:
            self.optlist, self.logfiles = getopt.getopt(argv[1:], self.optstr + caller_optstr)
        except getopt.error, s:
            print '%s: %s' % (self.caller, s)
            self.usage()
            sys.exit(1)

        optstr = ''
        print 'TCPGet opts:'
        for opt in self.optlist:

            if opt[0][1] in caller_optstr:
                caller_optlist.append(opt)
                continue
            if opt[0] == "-H":
                print
                print self.__doc__
                sys.exit(0)
            if opt[0] == "-v":
                set_print_full_hostname(1)
            if opt[0] == "-t":
                self.trace = 1
            if opt[0] == "-q":
                self.quiet = 1
                optstr += '-q'
            if opt[0] == "-r":
                self.nrecs = int(opt[1])
                optstr = optstr + '-r' + opt[1]
            if opt[0] == "-s":
                self.startconn = int(opt[1])
                optstr = optstr + '-s' + opt[1]
            if opt[0] == "-c":
                get_vals(opt[1], 'oneconn')
                optstr = optstr + '-c' + opt[1]
                print 'analysing only connection(s) ',
                print self.oneconn
            if opt[0] == '-C':
                get_hosts_wanted(opt[1], 'clis')
                optstr = optstr + '-C' + opt[1]
                print 'returning connections only for client(s) ',
                print opt[1]
            if opt[0] == '-S':
                get_hosts_wanted(opt[1], 'servs')
                optstr = optstr + '-S' + opt[1]
                print 'returning connections only for server(s) ',
                print opt[1]
            if opt[0] == '-P':
                get_vals(opt[1], 'ports')
                optstr = optstr + '-P' + opt[1]
                print 'returning connections only for port(s) ',
                print opt[1]
            if opt[0] == '-L':
                self.log = 1
                self.logdir = opt[1]
            if opt[0] == '-l':
                self.log = 1
            

        self.optstr = optstr

        return caller_optlist


############################################################################

	    
##############################################################################
##############################################################################

def main():

    """np_TCPGet.py provides the TCPGet class.

    When run as script provides trivial example of the class's use and
      examples of use of the Histogram class

    Usage: np_TCPGet.py [flags] <rep file list>
      Flags:
        -h - This documentation
        -H - documentation on np_TCPGet class
        -T - trace (verbose) mode
        -d - collect connection durations
        -e - collect connection set up durations
        -w - investigate occurence of zero window ads.
        -p - collect client/server payload sizes
        -i - print summary data on each connection
          (if one of d, e, w, p, i flags are not set default is to
            print connection id)
        -o - examine connections in open order (default is close order)
        -m - minimise memory usage if -o set

        ... and flags accepted by the np_TCPGet class

    """

    def nullf(s):

        pass

    from histo import Histogram, HistogramError, HistogramLowRangeError, HistogramHighRangeError
    from np_TCPDisplay import TcpDisplay

    # This is string of c.l. args accepted by this script
    optstr = 'Tdewphiom'

    scriptname = os.path.basename(sys.argv[0])

    # instantiate a connection getter
    getter = TCPGet(get_hdrs=0, open_order=0, min_mem=0)

    # pass flags to getter and collect what's left for us
    optlist = getter.set_args(sys.argv, optstr)

    trace = 0
    do_dur = 0
    do_est = 0
    do_zw = 0
    do_pl = 0
    print_id = 0
    print_conn = 0
    opord = 0
    minmem = 0

    # interpret our flags
    for opt in optlist:

        if opt[0] == '-T':
            trace = 1
            getter.trace = 1
        if opt[0] == '-d':
            do_dur = 1
        if opt[0] == '-e':
            do_est = 1
        if opt[0] == '-w':
            do_zw = 1
        if opt[0] == '-p':
            do_pl = 1
        if opt[0] == '-i':
            print_conn = 1
        if opt[0] == '-o':
            opord = 1
        if opt[0] == '-m':
            minmem = 1
        if opt[0] == '-h':
            print
            print main.__doc__
            sys.exit(0)

    nboth = 0
    no_c = 0
    no_s = 0

    if not (do_dur or do_est or do_zw or do_pl or print_conn):
        print_id = 1

    #
    # c.l. flags may have changed getter parameters - reset
    #
    getter.set_ops(open_order=opord, min_mem=minmem)


    #
    # Set up histograms to collect required data
    #
    if do_dur:
        try:
            #
            # set bounds based on first 5000 samples, save OOB samples for
            # examination
            #
            dh = Histogram(ntrigvals=5000, save_orange=1)
        except HistogramError, s:
            print s, 'Duration histogram'
            return

    if do_est:
        try:
            #
            # set upper bound based on first 5000 samples and lower bound of
            # zero. Raise OOB exceptions
            #
            eh = Histogram(lower=0, ntrigvals=5000, rangeexcept=1)
        except HistogramError, s:
            print s, 'Establishment histogram'
            return
        #
        # set to get hdrs - may want to see what's going on if low range
        # exceptions raised
        #
        getter.set_ops(get_hdrs=1)

    if do_zw:
        try:
            # good idea of what's to be catered for here
            zh = Histogram(lower=0, bucketsz=1, upper=10)
        except HistogramError, s:
            print s, 'ZW histogram'
            return
        getter.set_ops(get_hdrs=1)

    if do_pl:
        try:
            phc = Histogram(lower=0, bucketsz=100, ntrigvals=1000, save_orange=1)
            phs = Histogram(lower=0, bucketsz=100, ntrigvals=1000, save_orange=1)
        except HistogramError, s:
            print s, 'PL histogram'
            return

    #
    # Here's the main loop to get the connections
    #
    for c in getter.get_conns():
        if print_id:
            print '#%d\n' % (c.id)
        if print_conn:
            c.printself_with_pkts()
            #c.printself()

        #
        # Discard conns where both client and server not seen
        #
        flags = c.flags

        if not flags & TCP_CLIENT_SEEN:
            no_c += 1
            continue
        if not flags & TCP_SERVER_SEEN:
            no_s += 1
            continue

        nboth += 1
        if do_dur:
            dh.add(c.dur/1000)
            
        if do_est:
            # have to catch OOB exceptions for this one
            try:
                eh.add((c.cliacksyn - c.clisyn)/1000)
            except HistogramLowRangeError, e:
                print e.val
                # take a look at this connection
                c.printself_with_pkts()
                # or alternatively could visualise it
                #TcpDisplay([c], standalone='yes', path='',logfun=nullf, trace=1)

        if do_zw:
            pl = c.pktlist
            zw = 0
            nzw = 0
            for p in pl:
                if p.window == 0:
                    if not zw:
                        zw = 1
                        nzw += 1
                else:
                    zw = 0
            zh.add(nzw)

        if do_pl:
            phc.add((c.cbytes, c.id))
            phs.add((c.sbytes, c.id))

    print '%d connections' % (nboth)
    print 'No server seen %d' % (no_s)
    print 'No client seen %d' % (no_c)

    #
    # Now look at the results collected
    #

    #draw the first two if done
    if do_dur:
        dh.results(draw=1, title='Connection duration (ms)')
        
    if do_est:
        eh.results(draw=1, title='Connection establishment (ms)')

    # just print out these results
    if do_zw:
        zh.results(printit=1, title='No. window closures')

    # have a different look at some of the data for these
    if do_pl:
        for h, end in [(phc, 'Client'), (phs, 'Server')]:
            print '%s results' % (end)
            d = h.results(draw=1, title='%s Payload' % (end))
            print '%s payloads (bytes):-' % (end)
            print '%d samples' % d['nsamples']
            print 'Min %d, Max %d, Upper histo bound %d - %d above as follows' % (d['min'], d['max'], d['upper'], d['nhigh'])
            for b in d['highs']:
                print '\t#%5d %10d' % (b[1], b[0])
            print
    
	    
##############################################################################
##############################################################################


# Call main when run as script
if __name__ == '__main__':
        main()