wread_util.c

该软件根据网络数据生成NetFlow记录。NetFlow可用于网络规划、负载均衡、安全监控等

/*  -*- Mode: C;  -*- */

/******************************************************************************
*                                                                             *
*   Copyright 2005 University of Cambridge Computer Laboratory.               *
*                                                                             *
*   This file is part of Nprobe.                                              *
*                                                                             *
*   Nprobe is free software; you can redistribute it and/or modify            *
*   it under the terms of the GNU General Public License as published by      *
*   the Free Software Foundation; either version 2 of the License, or         *
*   (at your option) any later version.                                       *
*                                                                             *
*   Nprobe is distributed in the hope that it will be useful,                 *
*   but WITHOUT ANY WARRANTY; without even the implied warranty of            *
*   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the             *
*   GNU General Public License for more details.                              *
*                                                                             *
*   You should have received a copy of the GNU General Public License         *
*   along with Nprobe; if not, write to the Free Software                     *
*   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA *
*                                                                             *
******************************************************************************/


#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <errno.h>
#include <getopt.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <unistd.h>
#include <sys/time.h>
#include <sys/resource.h>
#ifdef __alpha__
#include <sys/mbuf.h>
#endif
#include <net/route.h>
#include <net/if.h>
#include <netdb.h>
#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#ifdef __alpha__
#include <netinet/ip_var.h>
#endif
#include <netinet/tcp.h>
#include <netinet/if_ether.h>
#include <arpa/inet.h>
#include <signal.h>
#include <limits.h>
#include <assert.h>

#include "basic_defs.h"

#include "list.h"

#include "malloc_rec.h"

#include "pkt.h"
#include "seq.h"
#include "flows.h"
#include "http.h"
#include "tcp.h"
#include "service.h"
#include "udp.h"
#include "udp_ns.h"
#include "counters.h"
#include "icmp.h"
#include "print_util.h"
#include "sundry_records.h"
#include "if_stats.h"
#include "procstat.h"
#include "report.h"
#include "output.h"

#include "servers.h"
#include "wread_util.h"
#include "print_util.h"
#include "interesting.h"
#include "content_t.h"
#include "np_file.h"
#include "if_stats.h"

#ifdef SWIG
#include "except.h"
#endif

char *prog = NULL;
char errbuf[100];
long long start_us;
int ncpus;          /* of collecting machine */
int hz;
int pgsz;


void 
wr_error(char *msg)
{
  if (prog)
    fprintf(stderr, "%s: ERROR ", prog);
  else
    fprintf(stderr, "ERROR ");
  //printf("%s: ERROR ", prog);
  if (errno)
    perror(msg);
  else
    fprintf(stderr, "%s\n", msg);
//printf("%d\n", 1/0);
  exit(1);
}

/****************************************************************************/


/*
 * Applicable to file as a whole 
 */

int 
_file_open(struct np_file *file)
{
  char errbuf[100];


  if (!strcmp("-", file->fnm))
    file->file = stdin;
  else if ((file->file = fopen(file->fnm, "r")) == NULL)
    {
      sprintf(errbuf, "can't open %s", file->fnm);
      ERROR(errbuf);
    }

  clearerr(file->file);

  /* get counters and included file header */
  if (fseek(file->file, -sizeof(counters_t), SEEK_END) != 0)
      ERROR("file_open - fseek() counters failed");
  GET_STRUCT(file->file, &file->counters, counters_t);

  if (file->counters.fh.magic != WAN_MAGIC)
    {
      fprintf(stderr, "File %s\n", file->fnm);
      ERROR("file_open - can't locate counters");
    }

  /* Check version */
  if (file->counters.fh.vers_pre != VERS_PRE 
      || file->counters.fh.vers_n != VERS_N)
    {
      char errbuf[256];
      sprintf(errbuf, "Attempt to open Wan Rep file with wrong version reader\nThis is version %c.%u - file is %c.%u", 
	      VERS_PRE, VERS_N,
	      file->counters.fh.vers_pre, file->counters.fh.vers_n);
      wr_error(errbuf);
      /* NOT REACHED */
    }

  /* set log start time global */
  start_us = file->counters.start.tv_sec*US_IN_S_LL;
  /* printf("START_US = %lld\n", start_us); */

  /* set ncpus, hz globals */
  ncpus = file->counters.fh.ncpus;
  hz = file->counters.fh.hz;
  pgsz = file->counters.fh.pgsz;

  /*
   * set file sizes (for backwards compatability fall back on constants if not
   *  saved in log)
   * 
   * TODO - all configuration parameters should be collected together
   *  and saved in log
   * 
   */

  if (file->counters.fh.rep_fsz == 0)
    {
      file->counters.fh.rep_fsz = REPORT_FILE_SZ;
      file->counters.fh.dump_fsz = DUMP_FILE_SZ;
    }


  /* position to read records */
  if (fseek(file->file, 0, SEEK_SET) != 0)
      ERROR("file_open - fseek() records failed");

  file->patches = 0;

#ifdef DUMP_DEBUG
  file->indx = 0;
#endif

  return 0;
}

int 
_file_reset(struct np_file *file)
{
  clearerr(file->file);
  /* position to read records */
  if (fseek(file->file, 0, SEEK_SET) != 0)
      ERROR("file_open - fseek() records failed");

  file->patches = 0;

#ifdef DUMP_DEBUG
  file->indx = 0;
#endif

  return 0;
}


/*
 * Print entire contents - optionally of type X
 */

int _printfile(struct np_file *file, int allocflag, unsigned char type, 
	       int keytype, unsigned int key, int quietflag)

{
  int i;
  int ret = 0;
  int rec_type;
  //unsigned int conn_id;
  tcp_hdrs_t hdrs;
  tcp_conn_t tconn;
  udp_conn_t uconn;
  sk_buff_alloc_fail_rec_t buf_fail;
  period_report_t prep;
  np_rusage_t ru;
  flow_inner_t flow;
  tcp_open_t flow_open;
  ns_fullrec_t ns_rec;
  struct icmp_rec irec;
  wrapper_record_t wrapper;
  procstat_rec_t pr;
  net_device_stats_t nicerrs;
  char rrbuf[NS_MAX_RRBUFSZ];
  int way;
  char intbuf[INTBUF_LEN_MAX + 1];
  int kt = keytype;

  char errbuf[100];

  _rewind(file);

  /* set up transaction chain */
  if (allocflag == TRANS_PREALLOC)
    {
      _http_alloc_trans(&tconn, MAX_NTRANS, PROVIDE_IMGBUFS);
      _tcp_alloc_hdrbuffs(&tconn.hdrs, MAX_TCP_DUMPHDRS_HELD);
      _tcp_alloc_hdrbuffs(&hdrs, MAX_TCP_DUMPHDRS_HELD);
    }

  ns_rec.rrbuf = rrbuf;
  
  /* run through connection records */
  for (;;)
    {
      if ((rec_type = _next_rec(file, type)) == -1)
	break;
      i = file->indx;
      //fprintf(stderr, "reading #%ld\n", i);

      if (kt == KEY_REC)
	{
	  if (i-1 < key)
	    {
	      _advance(file);
	      continue;
	    }
	  else if (i-1 == key)
	    {
	      keytype = 0;
	    }
	  else
	    {
	      break;
	    }
	}

      //printf("%d\n", rec_type);

      switch(rec_type)
	{
	case REC_TCP_TEST_OPEN:
	case REC_TCP_HTTP_OPEN:
	case REC_TCP_FTP_OPEN:
	case REC_TCP_FTP_DATA_OPEN:
	case REC_TCP_RTSP_OPEN:
	case REC_TCP_BGP_OPEN:
	case REC_TCP_PNM_OPEN:
	case REC_TCP_OTHER_OPEN:
	  _read_tcp_open(file, &flow_open);
	  if (!quietflag)
	    if (!keytype 
		|| (keytype == KEY_CONN_ID && flow_open.conn_id == key)
		|| (keytype == KEY_SERV_ADDR && flow_open.flow.dstaddr == key)
		|| (keytype == KEY_CLI_ADDR && flow_open.flow.srcaddr == key))
	      report_tcp_open(&flow_open, flow_open.flow.first_arr_tm, file->indx -1);
	  break;

	case REC_TCP_TEST_HDRS:
	case REC_TCP_HTTP_HDRS:
	case REC_TCP_FTP_HDRS:
	case REC_TCP_FTP_DATA_HDRS:
	case REC_TCP_RTSP_HDRS:
	case REC_TCP_BGP_HDRS:
	case REC_TCP_PNM_HDRS:
	case REC_TCP_OTHER_HDRS:
	  _read_tcp_hdrs(file, &hdrs, allocflag);
	  if (!quietflag)
	    if (!keytype 
		|| (keytype == KEY_CONN_ID && hdrs.conn_id == key)
		|| (keytype == KEY_SERV_ADDR && flow_open.flow.dstaddr == key)
		|| (keytype == KEY_CLI_ADDR && flow_open.flow.srcaddr == key))
	      report_tcp_hdrs(&hdrs, file->indx -1);
	  break;

	case REC_TCP_TEST:
	case REC_TCP_HTTP:
	case REC_TCP_FTP:
	case REC_TCP_FTP_DATA:
	case REC_TCP_RTSP:
	case REC_TCP_BGP:
	case REC_TCP_PNM:
	case REC_TCP_OTHER:
	  _read_tcp_conn(file, &tconn, TRANS_PREALLOC, GET_TRANS);
	  if (!quietflag)
	    if (!keytype 
		|| (keytype == KEY_CONN_ID && tconn.hdrs.conn_id == key)
		|| (keytype == KEY_SERV_ADDR 
		    && tconn.flow_inner.dstaddr == key)
		|| (keytype == KEY_CLI_ADDR 
		    && tconn.flow_inner.srcaddr == key))
	      report_tcp_conn(stdout, &tconn, file->indx -1, 1);
	  break;
	  
	case REC_UDP_NFS:
	case REC_UDP_ICQ:
	case REC_UDP_OTHER:
	  uconn.service_data = NULL;
	  _read_udp_conn(file, &uconn, DATA_PREALLOC, GET_DATA);
	  if (!quietflag && !keytype)
	    report_udp_conn(&uconn, file->indx -1);
	  break;

	case REC_UDP_DNS:
	  uconn.service_data = &ns_rec;
	  _read_udp_conn(file, &uconn, DATA_PREALLOC, GET_DATA);
	  if (!quietflag && !keytype)
	    report_udp_conn(&uconn, file->indx -1);
	  break;

	case REC_ICMP_UNREACH:
	  _read_icmp(file, &irec);
	  if (!quietflag && !keytype)
	    report_icmp(&irec, file->indx -1);
	  break;

	case REC_BUF_ALLOC_FAIL:
	  _read_buf_alloc_fail(file, &buf_fail);
	  if (!quietflag && !keytype)
	    report_buf_alloc_fail(&buf_fail, file->indx -1);
	  break;

	case REC_NIC_FAIL:
	  _read_nic_fail(file, &nicerrs);
	  if (!quietflag && !keytype)
	    report_nic_fail(&nicerrs, file->indx -1);
	  break;

	case  REC_PERIOD_REPORT:
	  /* also attempts to read a following ru record */
	  ret = _read_period_report(file, &prep, &ru);
	  if (!quietflag && !keytype)
	    {
	      report_period_report(&prep, file->indx -2, &file->counters, REC_PERIOD_REPORT);
	      if (!ret)
		report_rusage(&ru, file->indx -1, 0);
	    }
	  
	  break;

	case  REC_WIRE_PERIOD_REPORT:
	  if (type == REC_WIRE_PERIOD_REPORT)
	    {
	      ret = _read_period_report(file, &prep, NULL);
	      report_period_report(&prep, file->indx, &file->counters, REC_WIRE_PERIOD_REPORT);
	    }
	  else
	    {
	      /* just clear it */
	      _advance(file);
	    }
	  break;

	case REC_RUSAGE:
	  _read_rusage(file, &ru);
	  if (!quietflag && !keytype)
	    report_rusage(&ru, file->indx -1, 1);
	  break;

	case REC_INTERESTING:
	  _read_interesting(file, &flow, &way, intbuf);
	  if (!quietflag && !keytype)
	    report_interesting(&flow, way, intbuf, file->indx -1);
	  break;

	case REC_INFORM:
	  _read_inform(file, intbuf);
	  if (!quietflag && !keytype)
	    report_inform(intbuf, file->indx -1);
	  break;

	case REC_OTHER_PROCSTAT:
	  _read_procstats(file, &pr);
	  if (!quietflag && !keytype)
	    report_procstats(&pr, ncpus, hz, pgsz, file->indx -1);
	  break;

	case REC_OTHER_WRAPPER:
	  _read_wrapper(file, &wrapper);
	  if (!quietflag && !keytype)
	    report_wrapper(&wrapper, file->indx -1);
	  break;
	  

	default:
	  sprintf(errbuf, "_printfile(): unknown record type (%d) encountered at record %d file %s", rec_type, file->indx -1, file->fnm);
	  ERROR(errbuf);
	} /* end switch */

      if (ret == -1)
	/* EOF from a _read */
	break;
    }

  if (allocflag == TRANS_PREALLOC)
    {
      /* free transaction chain */
      _http_dealloc_trans(&tconn);
      _tcp_dealloc_hdrbuffs(&tconn.hdrs);
      _tcp_dealloc_hdrbuffs(&hdrs);
    }

  return 0;
}

/*
 * Re-write log file to include only counters and desired record type
 */

#define RECBUF_SZ 16384

int 
_filter_file(struct np_file *filein, int type, char *outfile)
{
  FILE *ofile, *ifile = filein->file;
  int rec_type, len;
  struct rep_rec_hdr rec_hdr;
  unsigned int indx = 0UL;
  char recbuff[RECBUF_SZ];
  

  if ((ofile = fopen(outfile, "w")) == NULL)
    wr_error("_filter_file: fopen()");


  _rewind(filein);

  rec_hdr.magic = REC_HDR_MAGIC;
  
  /* run through connection records */
  for (;;)
    {
      
      if ((rec_type = _next_rec(filein, type)) == -1)
	break;

      len = filein->curr_len;
      
      rec_hdr.indx = indx;
      rec_hdr.type = (unsigned char)(rec_type & 0xff);
      rec_hdr.len = (unsigned int)(len+sizeof(struct rep_rec_hdr));
      
      if (fwrite(&rec_hdr, sizeof(struct rep_rec_hdr), 1, ofile) 
	  != 1)
	{
	  
	  fprintf(stderr, "%s\n", strerror(errno));
	  
	  ERROR("_filter_file: header fwrite()");
	}
      

      GET_MEM(ifile, recbuff, len);
      
      if (fwrite(recbuff, len, 1, ofile) != 1)
	wr_error("_filter_file: record fwrite()");

      indx++;
    }
  
  /*
   *  Now finish off with the counters
   */

  /* step back and check at counters - no need to read them */
   
  if (fseek(ifile, -sizeof(struct rep_rec_hdr), SEEK_CUR) != 0)
    ERROR("_filter_file(): fseek()");

  GET_STRUCT(ifile, &rec_hdr, struct rep_rec_hdr);
  if (feof(ifile))
    ERROR("_filter_file(): didn't find counters");

  filein->counters.nrecords = indx;
  
  rec_hdr.indx = indx;
  rec_hdr.type = REC_COUNTERS;
  rec_hdr.len = sizeof(counters_t);
  
  if (fwrite(&rec_hdr, sizeof(struct rep_rec_hdr), 1, ofile) 
      != 1)
    ERROR("_filter_file: counters header fwrite()");
  
  if (fwrite(&filein->counters, sizeof(counters_t), 1, ofile) 
      != 1)
    wr_error("_filter_file: counters fwrite()");

  if (fclose(ofile) != 0)