ftstat.c

netflow,抓包

/*
 * Copyright (c) 2002 Mark Fullmer and The Ohio State University
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *      $Id: ftstat.c,v 1.36 2003/04/04 02:26:20 maf Exp $
 */

#include "ftconfig.h"
#include "ftlib.h"

#include <sys/time.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/uio.h>
#include <sys/socket.h>
#include <sys/resource.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <sys/stat.h>
#include <ctype.h>
#include <stddef.h>
#include <syslog.h>
#include <dirent.h>
#include <limits.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <fcntl.h>
#include <zlib.h>

#if HAVE_STRINGS_H
 #include <strings.h>
#endif
#if HAVE_STRING_H
  #include <string.h>
#endif

#if !HAVE_STRSEP
  char    *strsep (char **, const char *);
#endif

#if HAVE_LL_STRTOUL
  #define strtoull strtoul
#endif /* HAVE_LL_STRTOULL */

#define FMT_SYM_LEN 64

/*
 * ****  Datastructures and other implementation notes  ***
 *
 * Each report requires
 *
 *   struct ftstat_rpt_n which is allocated dynamically at run time.
 *  
 *   ftstat_rpt_n_new() -   allocated ftstat_rpt_n
 *   ftstat_rpt_n_accum() - called for each flow
 *   ftstat_rpt_n_calc()  - final calculations after all flows
 *   ftstat_rpt_n_dump()  - report to file
 *   ftstat_rpt_n_free()  - free storage allocated by ftstat_rpt_n_new()
 *     and others during the report run.
 *
 * The reports are grouped by a definition.  Typically the caller will
 * invoke all reports in a group by calling ftstat_def_new(),
 * ftstat_def_accum(), ftstat_def_calc(), ftstat_def_dump(), and
 * ftstat_def_free().
 *
 * The reports in the configuration file are stored in a linked
 * list of ftstat_rpt with the head in struct ftstat.
 *
 * The definitions in the configuration file are stored in a linked
 * list of ftstat_def with the head in struct ftstat.
 *
 * Each definition contains a linked list of ftstat_rpt_item
 * which points to a report (struct ftstat_rpt).
 *
 * Definitions reference reports.  Initially the report name is stored
 * in the definition and then when EOF is reached resolve_reports()
 * is called to fill in the pointers.
 *
 * The config file can reference a filter file and a tag file.  These
 * are loaded when the first tag or filter is specified.
 *
 * Each report can have a filter and a definition can have a filter
 * for all the reports.  Configuration of both is allowed.
 *
 * Each definition can invoke a tag definition. (add tags)
 *
 * Each definition can invoke a mask definition. (replace masks)
 *
 * Each report type is defined by the enum ftstat_rpt_type
 * and then the struct typelookup (tlookup) is used to configure
 * all the other report specific data items such as its ASCII
 * name, fields required, function pointers to the report specific
 * functions, options supported, etc.
 *
 * Special note for the *ps (ie min_pps, max_pps, avg_pps, min_bps,
 *  max_bps, avg_bps) calculations.  A memory optimization exists that
 *  removes the sizeof struct ftps bytes from the end of all the hash
 *  and bucket allocations (for bucket, they're not allocated).  This
 *  means that struct _must_ exist or the subtraction bytes will end
 *  up trashing real data.  An example is ftchash_rec_int where the
 *  ftps would never be used, yet it's there so the code that subtracts
 *  it off works properly.
 *
 */

static struct fts3rec_offsets nfo;
static u_int64 *sort_i64;
static double *sort_idouble;

#define DUMP_STD_OUT()\
if (rpt->out->fields & FT_STAT_FIELD_INDEX) {\
  len += fmt_uint64(fmt_buf, rpt->idx++, FMT_JUST_LEFT);\
  comma = 1;\
}\
if (rpt->out->fields & FT_STAT_FIELD_FIRST) {\
  if (comma) fmt_buf[len++] = ',';\
  len += fmt_uint32(fmt_buf+len, rpt->time_start, FMT_JUST_LEFT);\
  comma = 1;\
}\
if (rpt->out->fields & FT_STAT_FIELD_LAST) {\
  if (comma) fmt_buf[len++] = ',';\
  len += fmt_uint32(fmt_buf+len, rpt->time_end, FMT_JUST_LEFT);\
  comma = 1;\
}\


#define CHASH_DUMP_INIT(A,B)\
  struct A *B;\
  struct tally tally;\
  char fmt_buf1[32], fmt_buf[1024];\
  int len, fmt, sort_flags, sort_offset, comma;\
  fmt = FMT_JUST_LEFT;\
  fmt_buf1[0] = fmt_buf[0] = 0;\
  bzero(&tally, sizeof tally);\
  tally.t_recs = ftch->entries;\
  if (rpt->out->options & FT_STAT_OPT_NAMES)\
    fmt |= FMT_SYM;\


#define CHASH_DUMP_STD_SORT(A) \
    if (rpt->out->sort_field == FT_STAT_FIELD_FLOWS) {\
      sort_offset = offsetof(struct A, nflows);\
      sort_flags |= FT_CHASH_SORT_64;\
    } else if (rpt->out->sort_field == FT_STAT_FIELD_OCTETS) {\
      sort_offset = offsetof(struct A, noctets);\
      sort_flags |= FT_CHASH_SORT_64;\
    } else if (rpt->out->sort_field == FT_STAT_FIELD_PACKETS) {\
      sort_offset = offsetof(struct A, npackets);\
      sort_flags |= FT_CHASH_SORT_64;\
    } else if (rpt->out->sort_field == FT_STAT_FIELD_DURATION) {\
      sort_offset = offsetof(struct A, etime);\
      sort_flags |= FT_CHASH_SORT_64;\
    } else if (rpt->out->sort_field == FT_STAT_FIELD_AVG_PPS) {\
      sort_offset = offsetof(struct A, ps.avg_pps);\
      sort_flags |= FT_CHASH_SORT_DOUBLE;\
    } else if (rpt->out->sort_field == FT_STAT_FIELD_MIN_PPS) {\
      sort_offset = offsetof(struct A, ps.min_pps);\
      sort_flags |= FT_CHASH_SORT_DOUBLE;\
    } else if (rpt->out->sort_field == FT_STAT_FIELD_MAX_PPS) {\
      sort_offset = offsetof(struct A, ps.max_pps);\
      sort_flags |= FT_CHASH_SORT_DOUBLE;\
    } else if (rpt->out->sort_field == FT_STAT_FIELD_AVG_BPS) {\
      sort_offset = offsetof(struct A, ps.avg_bps);\
      sort_flags |= FT_CHASH_SORT_DOUBLE;\
    } else if (rpt->out->sort_field == FT_STAT_FIELD_MIN_BPS) {\
      sort_offset = offsetof(struct A, ps.min_bps);\
      sort_flags |= FT_CHASH_SORT_DOUBLE;\
    } else if (rpt->out->sort_field == FT_STAT_FIELD_MAX_BPS) {\
      sort_offset = offsetof(struct A, ps.max_bps);\
      sort_flags |= FT_CHASH_SORT_DOUBLE;\
    } else {\
      fterr_errx(1,"chash_xxx_dump(): internal error");\
    }

#define CHASH_STD_OUT(A,B)\
      if ((rpt->out->options & FT_STAT_OPT_TALLY) && tally.rt_recs &&\
        (!(tally.rt_recs % rpt->out->tally))) {\
        if (rpt->all_fields & FT_STAT_FIELD_PS)\
          fprintf(fp, "#TALLY %%recs=%3.3f %%flows=%3.3f %%octets=%3.3f %%packets=%3.3f %%avg-bps=%3.3f %%avg-pps=%3.3f\n",\
            ((double)tally.rt_recs/(double)tally.t_recs)*100,\
            ((double)tally.rt_flows/(double)rpt->t_flows)*100,\
            ((double)tally.rt_octets/(double)rpt->t_octets)*100,\
            ((double)tally.rt_packets/(double)rpt->t_packets)*100,\
            (((double)tally.ravg_bps/(double)tally.rt_frecs)/\
              (double)rpt->avg_bps)*100,\
            (((double)tally.ravg_pps/(double)tally.rt_frecs)/\
              (double)rpt->avg_pps)*100);\
         else\
          fprintf(fp, "#TALLY %%recs=%3.3f %%flows=%3.3f %%octets=%3.3f %%packets=%3.3f\n",\
            ((double)tally.rt_recs/(double)tally.t_recs)*100,\
            ((double)tally.rt_flows/(double)rpt->t_flows)*100,\
            ((double)tally.rt_octets/(double)rpt->t_octets)*100,\
            ((double)tally.rt_packets/(double)rpt->t_packets)*100);\
\
      } /* tally */\
      tally.rt_flows += A->nflows;\
      tally.rt_octets += A->noctets;\
      tally.rt_packets += A->npackets;\
      tally.rt_recs ++;\
      tally.rt_frecs += A->nrecs;\
      if (rpt->all_fields & FT_STAT_FIELD_PS) {\
        tally.ravg_bps += A->ps.avg_bps * A->nrecs;\
        tally.ravg_pps += A->ps.avg_pps * A->nrecs;\
      }\
\
    if (rpt->out->fields & FT_STAT_FIELD_FLOWS) {\
      if (B) fmt_buf[len++] = ',';\
      len += fmt_uint64(fmt_buf+len, A->nflows, FMT_JUST_LEFT);\
      comma = 1;\
    }\
    if (rpt->out->fields & FT_STAT_FIELD_OCTETS) {\
      if (comma) fmt_buf[len++] = ',';\
      len += fmt_uint64(fmt_buf+len, A->noctets, FMT_JUST_LEFT);\
      comma = 1;\
    }\
    if (rpt->out->fields & FT_STAT_FIELD_PACKETS) {\
      if (comma) fmt_buf[len++] = ',';\
      len += fmt_uint64(fmt_buf+len, A->npackets, FMT_JUST_LEFT);\
      comma = 1;\
    }\
    if (rpt->out->fields & FT_STAT_FIELD_DURATION) {\
      if (comma) fmt_buf[len++] = ',';\
      len += fmt_uint64(fmt_buf+len, A->etime, FMT_JUST_LEFT);\
      comma = 1;\
    }\
    if (rpt->out->fields & FT_STAT_FIELD_AVG_BPS) {\
      if (comma) fmt_buf[len++] = ',';\
      len += sprintf(fmt_buf+len, "%f", A->ps.avg_bps);\
      comma = 1;\
    }\
    if (rpt->out->fields & FT_STAT_FIELD_MIN_BPS) {\
      if (comma) fmt_buf[len++] = ',';\
      len += sprintf(fmt_buf+len, "%f", A->ps.min_bps);\
      comma = 1;\
    }\
    if (rpt->out->fields & FT_STAT_FIELD_MAX_BPS) {\
      if (comma) fmt_buf[len++] = ',';\
      len += sprintf(fmt_buf+len, "%f", A->ps.max_bps);\
      comma = 1;\
    }\
    if (rpt->out->fields & FT_STAT_FIELD_AVG_PPS) {\
      if (comma) fmt_buf[len++] = ',';\
      len += sprintf(fmt_buf+len, "%f", A->ps.avg_pps);\
      comma = 1;\
    }\
    if (rpt->out->fields & FT_STAT_FIELD_MIN_PPS) {\
      if (comma) fmt_buf[len++] = ',';\
      len += sprintf(fmt_buf+len, "%f", A->ps.min_pps);\
      comma = 1;\
    }\
    if (rpt->out->fields & FT_STAT_FIELD_MAX_PPS) {\
      if (comma) fmt_buf[len++] = ',';\
      len += sprintf(fmt_buf+len, "%f", A->ps.max_pps);\
      comma = 1;\
    }\
    if (rpt->out->fields & FT_STAT_FIELD_FRECS) {\
      if (comma) fmt_buf[len++] = ',';\
      len += fmt_uint64(fmt_buf+len, A->nrecs, FMT_JUST_LEFT);\
      comma = 1;\
    }\
    fmt_buf[len++] = '\n';\
    fmt_buf[len] = 0;\
    fputs(fmt_buf, fp);\
    if (rpt->out->records && (tally.rt_recs == rpt->out->records)) {\
      fprintf(fp, "# stop, hit record limit.\n");\
      break;\
    }

#define CHASH_STDP_OUT(A,B)\
      if ((rpt->out->options & FT_STAT_OPT_TALLY) && tally.rt_recs &&\
        (!(tally.rt_recs % rpt->out->tally))) {\
        if (rpt->all_fields & FT_STAT_FIELD_PS)\
          fprintf(fp, "#TALLY %%recs=%3.3f %%flows=%3.3f %%octets=%3.3f %%packets=%3.3f %%avg-bps=%3.3f %%avg-pps=%3.3f\n",\
            ((double)tally.rt_recs/(double)tally.t_recs)*100,\
            ((double)tally.rt_flows/(double)rpt->t_flows)*100,\
            ((double)tally.rt_octets/(double)rpt->t_octets)*100,\
            ((double)tally.rt_packets/(double)rpt->t_packets)*100,\
            (((double)tally.ravg_bps/(double)tally.rt_frecs)/\
              (double)rpt->avg_bps)*100,\
            (((double)tally.ravg_pps/(double)tally.rt_frecs)/\
              (double)rpt->avg_pps)*100);\
         else\
          fprintf(fp, "#TALLY %%recs=%3.3f %%flows=%3.3f %%octets=%3.3f %%packets=%3.3f\n",\
            ((double)tally.rt_recs/(double)tally.t_recs)*100,\
            ((double)tally.rt_flows/(double)rpt->t_flows)*100,\
            ((double)tally.rt_octets/(double)rpt->t_octets)*100,\
            ((double)tally.rt_packets/(double)rpt->t_packets)*100);\
\
      } /* tally */\
      tally.rt_flows += A->nflows;\
      tally.rt_octets += A->noctets;\
      tally.rt_packets += A->npackets;\
      tally.ravg_bps += A->ps.avg_bps * A->nrecs;\
      tally.ravg_pps += A->ps.avg_pps * A->nrecs;\
      tally.rt_recs ++;\
      tally.rt_frecs += A->nrecs;\
\
  if (rpt->out->fields & FT_STAT_FIELD_FLOWS) {\
    if (B) fmt_buf[len++] = ',';\
    len += sprintf(fmt_buf+len, "%f",\
     ((double)A->nflows / (double)rpt->t_flows)*100.0);\
    comma = 1;\
  }\
  if (rpt->out->fields & FT_STAT_FIELD_OCTETS) {\
    if (comma) fmt_buf[len++] = ',';\
    len += sprintf(fmt_buf+len, "%f",\
     ((double)A->noctets / (double)rpt->t_octets)*100.0);\
    comma = 1;\
  }\
  if (rpt->out->fields & FT_STAT_FIELD_PACKETS) {\
    if (comma) fmt_buf[len++] = ',';\
    len += sprintf(fmt_buf+len, "%f",\
     ((double)A->npackets / (double)rpt->t_packets)*100.0);\
    comma = 1;\
  }\
  if (rpt->out->fields & FT_STAT_FIELD_DURATION) {\
    if (comma) fmt_buf[len++] = ',';\
    len += sprintf(fmt_buf+len, "%f",\
     ((double)A->etime / (double)rpt->t_duration)*100.0);\
    comma = 1;\
  }\
  if (rpt->out->fields & FT_STAT_FIELD_AVG_BPS) {\
    if (comma) fmt_buf[len++] = ',';\
    len += sprintf(fmt_buf+len, "%f",\
     ((double)A->ps.avg_bps / (double)rpt->avg_bps)*100.0);\
    comma = 1;\
  }\
  if (rpt->out->fields & FT_STAT_FIELD_MIN_BPS) {\
    if (comma) fmt_buf[len++] = ',';\
    len += sprintf(fmt_buf+len, "%f",\
     ((double)A->ps.min_bps / (double)rpt->min_bps)*100.0);\
    comma = 1;\
  }\
  if (rpt->out->fields & FT_STAT_FIELD_MAX_BPS) {\
    if (comma) fmt_buf[len++] = ',';\
    len += sprintf(fmt_buf+len, "%f",\
     ((double)A->ps.max_bps / (double)rpt->max_bps)*100.0);\
    comma = 1;\
  }\
  if (rpt->out->fields & FT_STAT_FIELD_AVG_PPS) {\
    if (comma) fmt_buf[len++] = ',';\
    len += sprintf(fmt_buf+len, "%f",\
     ((double)A->ps.avg_pps / (double)rpt->avg_pps)*100.0);\
    comma = 1;\
  }\