np_obtree.py

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

            elif rectype == REC_TCP_HTTP_HDRS: # a block of hdrs
                hdrs.get_hdrs_p(file)
                conn_id = hdrs.conn_id
                if conn_ids.has_key(conn_id):
                    conn_ids[conn_id].append((file.indx,
                                              REC_TCP_HTTP_HDRS, file, offset))

            elif rectype == REC_TCP_HTTP: #TCP/HTTP
                nread = nread + 1
                get_http_rec_and_trans(file, connrec, translist)
                # get the conn_id
                conn_id = connrec.get_conn_id()
                #print '#', conn_id
                
##                 # only want where both directions seen
##                 if not (connrec.server_seen() and connrec.client_seen()):
##                     if conn_ids.has_key(conn_id):
##                         # remove dictionary item
##                         del conn_ids[conn_id]
##                         ectrs.ow_rej += 1
##                     continue

                if conn_ids.has_key(conn_id): # one we're interested in
                    if connid and connid == conn_id:
                        done = 1
                    if tfilt.pss(connrec, translist): # one we're interested in
                        # find any headers already dumped
                        hdrlist = conn_ids[conn_id]
                        hdrlist.append((file.indx, REC_TCP_HTTP, file, offset))

                        scratch.conns_accepted += 1

                        # add a connection filerec to obdict
                        conn = FileRec('TCPConn', hdrlist)
                        #conn.add(hdrlist)
                        obdict[conn_id] = conn

                        # add conn to clidict
                        cli = connrec.shost()
                        try:
                            c = clidict[cli]
                        except KeyError:
                            c = clidict[cli] = []
                        c.append(conn_id)

                        # add conn to servdict
                        serv = connrec.dhost()
                        try:
                            s = servdict[serv]
                        except KeyError:
                            s = servdict[serv] = []
                        s.append(conn_id)

                    else:
                        # not interested
                        ectrs.tf_rej += 1
                    # remove dictionary item
                    del conn_ids[conn_id]

            elif rectype == REC_UDP_DNS: #UDP_DNS
                #
                # XX TMP XX
                #
                file.advance()
                continue
                #indx, uconnrec, nsrec = get_ns_rec(file)
                #do_ns(lookups, uconnrec, nsrec, cachefn)

            else:
                print '%s Illicit record type encountered %d' % (scriptname,
                                                                 rectype)
            if not nread%100:
                overprint('Record %d' % (nread))
##                 sys.stdout.write('.')
##                 sys.stdout.flush()

    overprint('%d relevant records' % (nread))

    print
    return ectrs

############################################################################

def find_delays(cr):

    return cr.has_delay()

############################################################################

#
# Global so accessible to exit function
#
namecache = None
stats = None
	
def main():
	
    global namecache, stats

    scriptname = os.path.basename(argv[0])
    draw_tree = 0
    nrecs = 0
    fspec = 0
    trace = 0
    verb = 0
    quiet = 0
    rec_obs = 0
    nob_thresh = NOB_THRESH
    tfilt = TFilter()
    clis = None
    servs = None
    cachedir = None
    connid = None
    startconn = 0
    wantfn = None
    pre_rank = None
    use_saved_delayed_servers = 0
    use_saved_delayed_servers_and_clients = 0
    savedata = 1
    report_mem = 0

    try:
        optlist, args = getopt.getopt(sys.argv[1:],
                                      'hvV:dtc:s:C:S:r:F:T:D:R:qN:x:mO')

    except getopt.error, s:
        print '%s: %s' % (scriptname, s)
        usage(scriptname)
        sys.exit(1)

    optstr = ''

    for opt in optlist:
        if opt[0] == "-h":
            usage(scriptname)
        if opt[0] == "-v":
            use_hostnames()
        if opt[0] == "-V":
            verb = 1
            if len(opt[1]):
                cachedir = opt[1]
        if opt[0] == "-d":
            draw_tree = 1
            trace = 1
        if opt[0] == "-q":
            quiet = 1
            optstr += '-q'
        if opt[0] == "-t":
            trace = 1
        if opt[0] == "-x":
            savedata = 0
        if opt[0] == "-r":
            nrecs = string.atoi(opt[1])
            optstr = optstr + '-r' + opt[1]
        if opt[0] == '-C':
            clis = get_clis_wanted(opt[1])
            optstr = optstr + '-C' + opt[1]
            print 'building trees only for client(s) ',
            print opt[1]
        if opt[0] == '-S':
            servs = get_clis_wanted(opt[1])
            optstr = optstr + '-S' + opt[1]
            print 'building trees only for server(s) ',
            print opt[1]
        if opt[0] == '-c':
            connid = int(opt[1])
            optstr = optstr + '-c' + opt[1]
            print 'building trees only for connid ',
            print opt[1]
	if opt[0] == "-m":
            write_err = sys.stderr.write
	    report_mem += 1
	if opt[0] == "-s":
	    startconn = string.atoi(opt[1])
	    optstr = optstr + '-s' + opt[1]
	if opt[0] == "-O":
	    rec_obs = 1
	if opt[0] == "-D":
            if opt[1] == 'p':
                wantfn = find_delays
            elif opt[1] == 's':
                use_saved_delayed_servers = 1
            elif opt[1] == 'ss':
                use_saved_delayed_servers_and_clients = 1
            else:
                print '-D: don\'t understand arg', opt[1]
                usage(scriptname)
	    optstr = optstr + '-D' + opt[1]
	if opt[0] == "-R":
            wantfn = 'rank' + opt[1]
	    optstr = optstr + '-R' + opt[1]
        if opt[0] == "-F":
            if opt[1] == 'h' or opt[1] == 'H':
                filter_help()
                sys.exit(0)
            fspec = string.atoi(opt[1])
            optstr = optstr + '-F' + opt[1]
        if opt[0] == "-T":
            if opt[1] == 'h' or opt[1] == 'H':
                tfilt.help()
                sys.exit(0)
            tfilt.build_filters(opt[1])
            optstr = optstr + '-T' + opt[1]
        if opt[0] == '-N':
            try:
                nob_thresh = int(opt[1])
            except:
                print 'Can\'t understand -N (nob_thresh) argument', opt[1]
                usage(scriptname)
            optstr = optstr + '-N' + opt[1] 

    if optstr and optstr[0] == '-':
        optstr = optstr[1:]
        
    signal(SIGINT, handle_sigint)

    #gc.disable()
    #gc.set_debug(gc.DEBUG_UNCOLLECTABLE)

    openfilelist, counters, basepath = get_files(args)
    basedir = os.path.dirname(basepath) + '/'
    run_start = tv2l(counters.run_start())

    if cachedir == '.' or cachedir == None:
        cachedir = basedir
    namecache = NameCache(cachedir, load=1)
    namecache.use_hostnames()
    lookup = namecache.hostname
    cachefn = namecache.add2cache

    if verb:
        lookup2 = lookup
    else:
        lookup2 = intoa_string

    atexit.register(exitfun)

    #counters.printself("")
    filepath = basepath + 'counters'
    counters.printself_tofile(filepath, '')
    del counters

    basep = basepath + optstr
    if os.path.basename(basep):
        basep += '.'
    basepath = basep + 'NOBT%d' % (nob_thresh)

    if draw_tree:
        logpath = '/dev/null'
    else:
        logpath = (basepath + '_treelog')


    # gc.disable()
    thresh = gc.get_threshold()
    print 'gc thresh =', thresh
    gc.set_threshold(350, 5, 5)
    #thresh = gc.get_threshold()
    #print 'gc thresh =', thresh
    #gc.set_debug(gc.DEBUG_LEAK)
    #gc.set_debug(gc.DEBUG_UNCOLLECTABLE | gc.DEBUG_INSTANCES )
    #gc.set_debug(gc.DEBUG_UNCOLLECTABLE | gc.DEBUG_INSTANCES | gc.DEBUG_OBJECTS | gc.DEBUG_SAVEALL)
    
    #
    # get a stats collector
    #
    invoke_args = ''
    for a in optlist:
        for aa in a:
            invoke_args += '%s' % (aa)
        invoke_args += ' '
    for a in args:
        invoke_args += '%s ' % (a)
    

    stats = TreeStats(invoke_args, quiet=quiet, savedata=savedata,
                      logpath=logpath, rec_obs=rec_obs)
    #stats.logpath = logpath
    stats.basepath = basep
    stats.basedir = basedir
    stats.lookup = lookup
    stats.lookup2 = lookup2
    stats.run_start = run_start
    write_log = stats.write_log
    
    stats.ectrs = E_Counters()
    stats.tfilt = tfilt
    stats.load_server_assoc()
    stats.nob_thresh = nob_thresh
    stats.open_savefiles()

    
    if wantfn:
        wantdicts = pre_pass_A(openfilelist, nrecs, wantfn, fspec, stats)
        print 'Using %d pre_passed delayed servers (%d clients)' % (len(wantdicts[0]), len(wantdicts[1]))
        lwd = len(wantdicts[0])
        strict_wantdict = None
        fspec = 0
    elif use_saved_delayed_servers:
        wantdicts = stats.get_delayed_servers()
        strict_wantdict = None
        print 'Using %d pre_saved delayed servers (%d clients)' % (len(wantdicts[0]), len(wantdicts[1]))
        lwd = len(wantdicts[0])
    elif use_saved_delayed_servers_and_clients:
        wantdicts = None
        strict_wantdict = stats.load_delayed_servers()
        print 'Using %d pre_saved delayed servers with clients' % (len(strict_wantdict))
        lwd = len(strict_wantdict)
    else:
        wantdicts = None
        strict_wantdict = None

    if wantdicts and wantdicts[0]:
        stats.add_server_assoc(wantdicts[0])
        print 'Added %d associated servers' % (len(wantdicts[0]) - lwd)

    if strict_wantdict:
        stats.add_strict_server_assoc(strict_wantdict)
        print 'Added %d associated servers' % (len(strict_wantdict) - lwd)

    #
    # Run through the logs, apply filters, build per client list of file references
    #
    pass1(openfilelist, stats, nrecs, clis, servs, wantdicts,
          strict_wantdict, connid, startconn, fspec, tfilt, cachefn)
    obdict = stats.obdict
    cdict = stats.cdict
    obdict['filepath'] = basepath
    del wantdicts
    
    #refs = [rec for rec in obdict.values() if rec.Class == 'WebClient']
    refs = [FileRec('WebClient', [obdict[c].recsets for c in cli]) \
            for cli in cdict.values()]

    print 'x', len(stats.sdict), 'servers'
    print 'x', len(stats.cdict), 'clients'
    print 'x', len(refs), 'refs'
    if quiet:
        del stats.sdict
        del stats.cdict
        del cdict

    stats.nclients = nrefs = len(refs)
    #print 'nrefs =', nrefs

    #print 'First collect'
    #print gc.collect()
    #raw_input('...')

    if report_mem:
        malloc_report()
        print

    #
    # Just fire up a window, construct clients from file records and display
    #
    if draw_tree:
        from np_TScreen import Tscreen
        print 'Pass 2: reconstructing clients prior to drawing'
        stats.ectrs.printself(tfilt)
        i = 1
        tlist = []
        lookups = stats.nslookups
        for c in refs:
            overprint('Constructing client %d of %d' % (i, nrefs))
            C = c.reconstruct(lookup=lookup)
            try:
                C.lookups = lookups[C.addr]
            except KeyError:
                # no NS lookups from this client
                pass
            tlist.append(C)
            i += 1
        Tscreen(tlist, lookup=lookup2, standalone='yes',
                path=basepath, trace=trace)

    #
    # Construct clients from file records, make trees, log significant findings
    #

    else:
        sys.stdout.flush()
        sys.stderr.write( 'Pass 2: reconstructing %d clients and building reference trees' % (nrefs))
        print
        for i in range(len(refs)):
            c = refs[i]
            overprint('Constructing client %d of %d - %d pages' % (i,
                                                      nrefs, stats.npages))
            try:
                C = c.reconstruct(lookup=lookup, logfun=write_log, trace=trace)
                C.make_tree(tmbase=run_start, logfun=write_log, trace=trace)
                stats.get_data(C)
            except:
                str = 'WebClient #%s page modelling failure:' % (C.addr_str())
                whoops(str)
                stats.write_log(str)
                if trace:
                    inform(str)
            C.tidy_up()
            if not i%100 and report_mem:
                sys.stdout.flush()
                write_err('Client %d of %d - %d pages\n' % (i, nrefs,
                                                          stats.npages))
                write_err('gc: '),
                write_err('len garbage list %d\n' % (len(gc.garbage)))
                #for g in gc.garbage:
                    #write_err(g.__str__())
                    #print g
                malloc_report()
                del gc.garbage[:]
                
                write_err('\n\n')
            if quiet:
                refs[i] = None
        sys.stdout.flush()
        sys.stderr.write('%d clients constructed' % (nrefs))
        print

        print
        print 'conns_accepted', scratch.conns_accepted
        print 'conn_recons', scratch.conn_recons
        print 'conns_intrees', scratch.conns_intrees

        stats.results()
	    
##############################################################################


# Call main when run as script
if __name__ == '__main__':
        main()