np_filerec.py

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

###############################################################################
#                                                                             #
#   Copyright 2005 University of Cambridge Computer Laboratory.               #
#                                                                             #
#   This file is part of Nprobe.                                              #
#                                                                             #
#   Nprobe is free software; you can redistribute it and/or modify            #
#   it under the terms of the GNU General Public License as published by      #
#   the Free Software Foundation; either version 2 of the License, or         #
#   (at your option) any later version.                                       #
#                                                                             #
#   Nprobe is distributed in the hope that it will be useful,                 #
#   but WITHOUT ANY WARRANTY; without even the implied warranty of            #
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the             #
#   GNU General Public License for more details.                              #
#                                                                             #
#   You should have received a copy of the GNU General Public License         #
#   along with Nprobe; if not, write to the Free Software                     #
#   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA #
#                                                                             #
###############################################################################


##############################################################################
##############################################################################
#
# Keep a record of Nprobe file records contributing to an object
#
###############################################################################
###############################################################################

#from string import rfind 
from os.path import split
#from print_col import *
from nprobe import tcp_conn, tcp_hdrs, rectype_str, intoa_string
from nprobe import REC_TCP_ALL, REC_TCP_HTTP_HDRS, REC_TCP_HTTP, HDRS_BIT
from  np_longutil import *
#from np_obnode import WebClient
#import np_obnode
from np_WebHost import WebHosts, WebClient, WebServer
from np_TCPConn import TCPConn, TCPHdr
from np_HTTPTrans import Transaction
from np_http_util import get_http_rec
import sys

import scratch


##############################################################################
##############################################################################

class FileRec:

    def __init__(self, type, recset, addr=None, rooturl=None,
                 lookup=intoa_string):

        self.Class = 'Filerec'
        self.type = type
        self.recsets = recset
        self.addr = addr
        self.rooturl = rooturl
        self.lookup = lookup

##############################################################################

    def __str__(self):


        str = 'FileRec for Class %s\n' % (self.type)
        if self.type.find('Web') == 0:
            for r in self.recsets:
                for rr in r:
                    # get last two elements of file path
                    sp = split(rr[2].fnm)
                    fnm = sp[0][sp[0].rfind('/')+1:] + '/' + sp[1]
                    str += '\t%s #%d rectype %s offset %d\n' % \
                       (fnm, rr[0], rectype_str(rr[1]), rr[3])
        elif self.type == 'TCPConn':
            for rr in self.recsets:
                # get last two elements of file path
                sp = split(rr[2].fnm)
                fnm = sp[0][sp[0].rfind('/')+1:] + '/' + sp[1]
                str += '\t%s #%d rectype %s offset %d\n' % \
                       (fnm, rr[0], rectype_str(rr[1]), rr[3])
        else:
            pass
        return str
                 
##############################################################################

    def logfun_null(self, s):

        pass

##############################################################################

    def add(self, reclist):

        self.recsets.append(reclist)

##############################################################################

    def reconstruct(self, logfun=None, trace=0, lookup=None):
        
        if not logfun:
            logfun = self.logfun_null

        if lookup:
            self.lookup = lookup
            
        if self.type == 'TCPConn':
            return self.build_TCPConn(self.recsets, logfun=logfun,
                                      trace=trace)

        elif self.type == 'WebClient':
            return self.build_webhost('client', self.lookup,
                                      clss= WebClient,
                                      logfun=logfun, trace=trace)
        elif self.type == 'WebServer':
            return self.build_webhost('server', self.lookup,
                                      clss= WebServer,
                                      logfun=logfun, trace=trace)
        elif self.type == 'WebHosts':
            return self.build_webhost('Class', self.lookup,
                                      clss= WebHosts,
                                      logfun=logfun, trace=trace)

##############################################################################

    #def build_WebClient(self, logfun=None, trace=0):
    def build_webhost(self, end, lookup, clss, logfun=None, trace=0):

        #print 'Reconstructing', clss

        iddict = {}
        TConns = []

        for set in self.recsets:
            TConns.append(self.build_TCPConn(set, logfun=logfun, trace=trace))

        addr = getattr(TConns[0], end)

        if self.addr:
            ad = self.addr
        else:
            ad = addr
        H = clss(ad)

        for c in TConns:
            #if getattr(c, end) != addr and clss != np_obnode.WebHosts:
            if getattr(c, end) != addr and clss != WebHosts:
                sys.stderr.write('Build_WebHost: %s addr mismatch\n' % (end))
                sys.exit(1)
            H.connlist.append(c)
            i = 0
            for t in c.tlist:
                t.connorder = i
                req = Transaction(t, c, H, lookup, logfun=logfun,
                                            trace=trace)
                #req.connorder = i
                if not req.valid:
                    continue
                H.translist.append(req)
                i += 1
            c.tlist = []
        H.FileRec = self
        H.rooturl = self.rooturl
                
        return H

##############################################################################

    def build_TCPConn(self, recset, logfun=None, trace=0):

        #print 'Reconstructing TCPConn - build_TCPConn', self.Class, reclist

        connrec, hdrlist, translist = self.build_tconn(recset)

        conn = TCPConn(connrec, hdrlist, logfun=logfun, trace=trace)
        conn.tlist = translist

        scratch.conn_recons += 1
        #conn.printself()

        if self.type.find('Web') == 0:
            conn.FileRec = FileRec('TCPConn', recset)
        else:
            conn.FileRec = self
        
        return conn
            

##############################################################################

    def build_tconn(self, reclist):

        #print 'Reconstructing TCPConn - build_tconn', reclist

        hlist = []
        nhdr = 0
                
        rec = reclist[-1]
        if rec[1] == REC_TCP_HTTP:
            indx, connrec, translist = self.get_tcp_http_conn(rec[2], rec[3])

        elif rec[1] == REC_TCP_ALL:
            indx, connrec, translist = self.get_tcp_conn(rec[2], rec[3])

        else:
            sys.stderr.write('Filerec.build_tconn: conn record type error %d\n' % (rec[1]))
            sys.exit(1)
            
        connrec.indx = rec[0] #indx returned is current serial read indx
        id = connrec.get_conn_id()

        if len(reclist) > 1:
            #print '  getting %d header sets' % (len(reclist)-1)
            for rec in reclist[:-1]:
                #print 'getting'
                if not rec[1] & HDRS_BIT:
                    sys.stderr.write('build_tconn: record type error %d\n' % (rec[1]))
                    print reclist
                    print rec
                    sys.exit(1)

                hdrs = self.get_tcp_hdrs(rec[2], rec[3])
                if hdrs.conn_id != id:
                    sys.stderr.write('Build_TCPconn: hdrs rec id error\n')
                    sys.exit(1)

                base_tm = ull2l(hdrs.get_hdrs_abstm())
                for i in range(hdrs.nheld):
                    hlist.append(TCPHdr(hdrs, i, base_tm, nhdr))
                    nhdr += 1
                    
        base_tm = ull2l(connrec.get_hdrs_abstm())
        for i in range(connrec.get_nhdrs_held()):
            hlist.append(TCPHdr(connrec, i, base_tm, nhdr))
            nhdr += 1
        
        return (connrec, hlist, translist)
            

##############################################################################

    def get_tcp_conn(self, file, offset):

        connrec = tcp_conn()

        # may be in midst of serial read - so reset to original offset
        mark = file.offset()
        
        file.seek(offset)
        connrec.get_conn(file)
        indx = connrec.indx
        connrec.indx = -1

        file.seek(mark)
        if file.offset() != mark:
            sys.stderr.write('Filerec.get_tcp_conn(): can\'t return to offset\n')
            sys.stderr.write('%s mark %d offset %d\n' % (file.fnm, mark, offset))
            sys.exit(1)
        return (indx, connrec, [])
            

##############################################################################

    def get_tcp_http_conn(self, file, offset):

        # may be in midst of serial read - so reset to original offset
        mark = file.offset()
        
        file.seek(offset)
        indx, connrec, translist =  get_http_rec(file)
        connrec.indx = -1

        file.seek(mark)
        if file.offset() != mark:
            sys.stderr.write('Filerec.get_tcp_http_conn(): can\'t return to offset\n')
            sys.stderr.write('%s mark %d offset %d\n' % (file.fnm, mark, offset))
            sys.exit(1)
        return (indx, connrec, translist)

##############################################################################

    def get_tcp_hdrs(self, file, offset):

        # may be in midst of serial read - so reset to original offset
        mark = file.offset()

        file.seek(offset)

        hdrs = tcp_hdrs(0)
        hdrs.get_hdrs(file)

        file.seek(mark)
        if file.offset() != mark:
            sys.stderr.write('Filerec.get_tcp_hdrs(): can\'t return to offset\n')
            sys.stderr.write('%s mark %d offset %d\n' % (file.fnm, mark, offset))
            sys.exit(1)

        return hdrs

##############################################################################
##############################################################################