gulp.c

改善linux指令 "tcpdump" 效能的免費open source程式

/*
 * Sniff the network and optionally decapsulate as Cisco ERSPAN packets.
 *
 * Because of its improved buffering and scheduling strategy, Gulp
 * should out-perform traditional capture programs such as tcpdump when
 * the goal is to capture and write to disk.  A 2.6GHz Intel core2duo CPU
 * running RHEL5 (linux 2.6.18) can capture and save to disk 1Gb/s
 * dropping 0 packets (for full-to-medium size packets).
 *
 *-----------------------------------------------------------------------
 *
 * Author: Corey Satten,  corey @ u.washington.edu, May 2007 - Mar 2008
 *
 * See http://staff.washington.edu/corey/gulp for more information and the
 * latest version.
 *
 *         Copyright (C) 2007 University of Washington
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 *-----------------------------------------------------------------------
 *
 * Usage: something like this to catch and decapsulate traffic from Cisco
 * remote span (ERSPAN) ports:
 *
 *     gulp -d ... > pcapfile
 *       or
 *     gulp -d ... | tcpdump -r - -w pcapfile ...
 *       or
 *     gulp -d ... | ngrep -I - -O pcapfile regexp ...
 *
 * or something like this to capture (and optionally filter) to a file:
 *
 *     gulp -f "optional pcap filter expression" ... > pcap_file
 *
 * or something like this to improve the performance of another sniffer:
 *
 *     tcpdump -i eth1 -s 0 -w - | gulp -c > pcap_file
 *
 * Gulp is threaded/buffered because writes to a file seem to sometimes
 * delay long enough to cause packet loss at the head of the pipeline even
 * though select would say the write will not block.
 *
 * Best results seem to come from confining the NIC interrupts and tcpdump
 * pipeline to CPU cores which share an L2 cache.  The former with
 *    # echo 3 > /proc/irq/#/smp_affinity   (see /proc/interrupts for #)
 * and the latter with
 *    # taskset -p 3 $$
 * see also http://staff.washington.edu/staff/corey/inter-core-benchmark
 *
 * The variables shared between threads are lock-free because each is
 * written only by one thread and careful coding ensures each thread
 * will always see a consistent-enough view to avoid problems.  This way
 * Gulp avoids locking overhead when the buffer is partly filled and
 * Gulp is working hardest at not dropping packets.  Traditional
 * signalling between threads could eliminate the short sleeps when the
 * buffer is either full or empty but these seem to consume negligible
 * time so why bother.
 */

#define _GNU_SOURCE
#ifdef linux
#include <syscall.h>
#endif
#include <unistd.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <pcap.h>
#include <strings.h>
#include <string.h>
#include <errno.h>
#include <signal.h>
#include <sched.h>
#include <sys/time.h>
#include <sys/file.h>
#include <sys/mman.h>
#include <sys/resource.h>
#include <fcntl.h>
#include <limits.h>

#define gettid() syscall(__NR_gettid)	/* missing in headers? */
#define RINGSIZE 1024*1024*100	/* about 5 seconds of data at 200Mb/s */
#define MAXPKT 16384		/* larger than any jumbogram */
#define WRITESIZE 65536		/* usual write chunk size - must be 2^N */
#define GRE_HDRLEN 50		/* Cisco GRE encapsulation header size */
#define SNAP_LEN 65535		/* apparently what tcpdump uses for -s 0 */
#define READ_PRIO	-15	/* niceness value for Reader thread */
#define WRITE_PRIO	10	/* niceness value for Writer thread */
#define READER_CPU	1	/* assign Reader thread to this CPU */
#define WRITER_CPU	0	/* assign Writer thread to this CPU */
#define POLL_USECS	1000	/* ring full/empty poll interval */
#ifdef RHEL3
# define my_sched_setaffinity(a,b,c) sched_setaffinity(a, c)
#else
# define my_sched_setaffinity(a,b,c) sched_setaffinity(a, b, c)
#endif /* RHEL3 */
#define V_WIDTH		10	/* minimum size of -V ps status field */
#define TEMPLATE "/gulp.XXXXXX"	/* mktemp template for files in -o dir */

#define RMEM_MAX "/proc/sys/net/core/rmem_max"		/* system tuning */
#define RMEM_DEF "/proc/sys/net/core/rmem_default"	/* system tuning */
#define RMEM_SUG 4194304				/* suggested value */
FILE *procf; int rmem_def=RMEM_SUG, rmem_max=RMEM_SUG;	/* check tuning */

int  WriteSize = WRITESIZE;	/* desired size for aligned writes */
int  snap_len = SNAP_LEN;	/* requested limit on packet capture size */
int  d_snap_len = SNAP_LEN;	/* actual limit on packet capture size */
int  poll_usecs = POLL_USECS;	/* ring full/empty poll interval */
int  just_copy = 0;		/* read from stdin instead of eth# */
int  captured = 0;		/* number of packets captured for stats */
int  ignored = 0;		/* number of packets !decapsulated for stats */
int  maxbuffered = 0;		/* maximum number of bytes ring buffered */
int  ringsize = RINGSIZE;	/* ring buffer size */
int  gre_hdrlen = 0;		/* decapsulation header length */
char *dev = "eth1";		/* capture interface device name */
char *filter_exp = "";		/* decapsulation filter expression */
char *buf;			/* pointer to the big malloc'd ring buffer */
int  volatile start, end;	/* index of first, next byte in buf */
int  volatile boundary = -2;	/* index in buf to start a new output file */
int  push, eof;			/* flags for inter-thread communication */
char *progname;			/* argv[0] for error messages from threads */
int  warn_buf_full = 1;		/* unless reading a file, warn if buf fills */
pcap_t *handle = 0;		/* packet capture handle */
struct pcap_stat pcs;		/* packet capture filter stats */
int got_stats = 0;		/* capture stats have been obtained */
char *id = "@(#) Gulp RCS $Revision: 1.58 $"; /* automatically maintained */
int  would_block = 0;		/* for academic interest only */
int  check_block = 0;		/* use select to see if writes would block */
int  yield_if_blocking = 0;	/* experimental: may help on uniprocessors */
char *ps_stat_ptr = 0;		/* loc to display buf percentage used */
int  ps_stat_len = 0;		/* initial length of -V arg */
int  xlock = 0;			/* set if exclusive lock requested */
int  lockfd;			/* open descriptor to file to lock */
char *odir = 0;			/* requested output directory name */
int  filec = 0;			/* output file number */
struct pcap_file_header fh;	/* begins every pcap file */
int  split_after = 10;		/* start new output file after # ringbufs */
int  max_files = 0;		/* upper bound on filec */
int  volatile reader_ready = 0;	/* reader thread no longer needs root */

/*
 * put data onto the end of global ring buffer "buf"
 */
void
append(char *ptr, int len, int bdry)
    {
    static int just_wrapped = 0;
    static int wrap_cnt = 0;
    int avail, used;
    static int warned = -1;
    used = end - start; if (used < 0) used += ringsize;
    if (used > maxbuffered) maxbuffered = used;
    avail = ringsize - used;

    while (len >= avail) {		/* ring buffer is full, wait */
	if (warned<push) {
	    warned = push;
	    if (warn_buf_full)
		fprintf(stderr, "%s: ring buffer full\n", progname);
	    }
	usleep(poll_usecs);
	used = end - start; if (used < 0) used += ringsize;
	avail = ringsize - used;
	if (eof) return;
	}
    if (len > 0 && len < avail) {	/* ring buffer space available */
	if (end + len <= ringsize) {	  /* no wrap to beginning needed */
	    memcpy(buf+end, ptr, len);
	    }
	else {				  /* append wraps */
	    int c = ringsize-end;	
	    memcpy(buf+end, ptr, c);
	    memcpy(buf, ptr+c, len-c);
	    }
	if (end+len >= ringsize) {
	    end += len-ringsize;
	    just_wrapped = 1;
	    }
	else {
	    end += len;
	    }
	if (just_wrapped && bdry) {
	    just_wrapped = 0;
	    if (odir && ++wrap_cnt >= split_after) {
		while (boundary >= 0) {	  /* last split still pending */
		    if (warned<push) {
			warned = push;
			if (warn_buf_full)
			    fprintf(stderr,"%s: ring buffer full\n", progname);
			}
		    usleep(poll_usecs);
		    }
		/*
		 * Tell Writer to start a new file.  Boundary is now < 0 so
		 * last split is complete.  Set boundary BEFORE appending file
		 * header; the write can't happen until the data is appended.
		 */
		boundary = end;
		wrap_cnt = 0;
		if (!just_copy) append((char *)&fh, sizeof(fh), 0);
		}
	    }
	}
    }

#ifndef JUSTCOPY
void
got_packet(u_char *args, const struct pcap_pkthdr *header, const u_char *packet)
    {
    struct pcap_pkthdr ph = *header;
    if (ph.caplen >= gre_hdrlen) {	/* sanity test */
	++captured;
	ph.caplen -= gre_hdrlen;
	ph.len -= gre_hdrlen;

#ifdef PCAP32_KLUDGE
	/*
	 * Because struct timeval is bigger on 64-bit linux than 32-bit
	 * linux and struct pcap_pkthdr has a struct timeval in it, pcap
	 * files generated by Gulp on 64-bit linux may be incompatible
	 * with programs expecting pcap files from 32-bit systems.  If
	 * you want Gulp to generate 32-bit compatible pcap files, first
	 * try compiling it after adding -m32 to CFLAGS in the Makefile
	 * (you may also need to install a 32-bit libpcap-devel package).
	 *
	 * If that doesn't work, try instead adding -DPCAP32_KLUDGE but
	 * note that the PCAP32_KLUDGE is not a complete solution
	 * because it will cause gulp to write files which it can't
	 * itself read (because reading is done by the pcap library code
	 * which will still be expecting 64-bit longs in struct timevals
	 * in packet headers).
	 */
	if (sizeof(long) > sizeof(int) && sizeof(int) > sizeof(short)) {
	    struct timeval_32 {
		int tv_sec;
		int tv_usec;
		} tv32;
	    tv32.tv_sec = ph.ts.tv_sec;
	    tv32.tv_usec= ph.ts.tv_usec;
	    append((char *)&tv32, sizeof(tv32), 0);
	    append((char *)&ph + sizeof(struct timeval),
		   sizeof(struct pcap_pkthdr) - sizeof(struct timeval), 0);
	    }
	else 
#endif /* PCAP32_KLUDGE */
	    append((char *)&ph, sizeof(struct pcap_pkthdr), 0);
	append((char *)packet+gre_hdrlen, ph.caplen, 1);
	}
    else ++ignored;
    }
#endif /* JUSTCOPY */

void
cleanup(int signo)
    {
    eof = 1;
    if (just_copy == 1 || got_stats) return;
#ifndef JUSTCOPY
#ifndef RHEL3
    pcap_breakloop(handle);
#endif
    if (pcap_stats(handle, &pcs) < 0) {
	if (strcmp(dev, "-"))	/* ignore message if input is stdin */
	    (void)fprintf(stderr, "pcap_stats: %s\n", pcap_geterr(handle));
	}
    else got_stats = 1;
#ifdef RHEL3
    pcap_close(handle);
#endif /* RHEL3 */
#endif /* JUSTCOPY */
    }

/*
 * This thread reads stdin or the network and appends to the ring buffer
 */
void *Reader(void *arg)
    {
#ifndef JUSTCOPY
    char errbuf[PCAP_ERRBUF_SIZE];	/* error buffer */
    struct bpf_program fp;		/* compiled filter program */
    bpf_u_int32 mask;			/* subnet mask */
    bpf_u_int32 net;			/* ip */
    int num_packets = -1;		/* number of packets to capture */
#endif
#ifdef CPU_SET
    int rtid = gettid();		/* reader thread id */
    cpu_set_t csmask;
    CPU_ZERO(&csmask);
    CPU_SET(READER_CPU, &csmask);
    if (my_sched_setaffinity(rtid, sizeof(cpu_set_t), &csmask) != 0) {
        fprintf(stderr, "%s: Reader could not set cpu affinity: %s\n",
	    progname, strerror(errno));
        }
    if (setpriority(PRIO_PROCESS, rtid, READ_PRIO) != 0) {
	fprintf(stderr, "%s: Reader could not set scheduling priority: %s\n",
	    progname, strerror(errno));
	}
#else
    replace with equivalent code for your OS or delete and run less optimally
#endif

#ifdef USE_SIGNAL
    signal(SIGINT, cleanup);
    signal(SIGPIPE, cleanup);
#else
    struct sigaction sa;
    sa.sa_handler = cleanup;
    sigemptyset(&sa.sa_mask);
    sa.sa_flags = 0;			/* allow signal to abort pcap read */
    
    sigaction(SIGINT, &sa, NULL);
    sigaction(SIGPIPE, &sa, NULL);
#endif /* USE_SIGNAL */

    if (just_copy) {
	static char rbuf[MAXPKT];
	int c;
	reader_ready = 1;
	while (!eof && (c = read(0, rbuf, MAXPKT)) != 0) {
	    if (c > 0) append(rbuf, c, 1);
	    }
	}
#ifndef JUSTCOPY
    else {

    /*
     * get network number and mask associated with capture device
     * (needed to compile a bpf expression).
     */
    if (strcmp(dev,"-") && pcap_lookupnet(dev, &net, &mask, errbuf) == -1) {
	fprintf(stderr, "%s: Couldn't get netmask for dev %s: %s\n",
	    progname, dev, errbuf);
	net = 0;
	mask = 0;
    }

    /* open capture device */
    if (!strcmp(dev, "-")) {
	handle = pcap_open_offline(dev, errbuf);
#ifndef RHEL3
	int sfd = -2;
	if (handle) sfd = pcap_get_selectable_fd(handle);
	if (sfd >= 0 && lseek(sfd, 0, SEEK_CUR) >= 0) {
	    warn_buf_full = 0;		/* input is a file, don't warn */
	    }
#endif /* RHEL3 */
	}
    else
	handle = pcap_open_live(dev, d_snap_len, 1, 0, errbuf);
    if (handle == NULL) {
	fprintf(stderr, "%s: Couldn't open device %s: %s\n",
	    progname, dev, errbuf);
	exit(EXIT_FAILURE);
    }

    reader_ready = 1;

    /* make sure we're capturing on an Ethernet device */
    if (pcap_datalink(handle) != DLT_EN10MB) {
	fprintf(stderr, "%s: %s is not an Ethernet\n", progname, dev);
	exit(EXIT_FAILURE);
    }

    /* compile the filter expression */
    if (pcap_compile(handle, &fp, filter_exp, 0, net) == -1) {
	fprintf(stderr, "%s: Couldn't parse filter %s: %s\n",
	    progname, filter_exp, pcap_geterr(handle));
	exit(EXIT_FAILURE);
    }

    /* apply the compiled filter */
    if (pcap_setfilter(handle, &fp) == -1) {
	fprintf(stderr, "%s: Couldn't install filter %s: %s\n",
	    progname, filter_exp, pcap_geterr(handle));
	exit(EXIT_FAILURE);
    }

    /*
     * emit pcap file header
     */

#ifndef RHEL3
    char tmpstr[] = "/tmp/gulp_hdr.XXXXXX";
    int tmpfd = mkstemp(tmpstr);
    if (tmpfd >= 0) {
	pcap_dumper_t *dump = pcap_dump_fopen(handle, fdopen(tmpfd,"w"));
	if (dump) pcap_dump_close(dump);
	tmpfd = open(tmpstr, O_RDONLY);	/* get pcap to create a header */
	if (tmpfd >= 0) read(tmpfd, (char *)&fh, sizeof(fh));
	if (tmpfd >= 0) close(tmpfd);
	unlink(tmpstr);
	fh.snaplen = snap_len;		/* snaplen after any decapsulation */
	}
#endif /* RHEL3 */
    if (fh.magic != 0xa1b2c3d4) {	/* if the above failed, do this */
	fprintf(stderr, "%s: using canned pcap header\n", progname);
	fh.magic = 0xa1b2c3d4;
	fh.version_major = 2;
	fh.version_minor = 4;
	fh.thiszone = 0;
	fh.sigfigs = 0;
	fh.snaplen = snap_len;
	fh.linktype = 1;
	}

    append((char *)&fh, sizeof(fh), 0);

    /* now we can set our callback function */
    pcap_loop(handle, num_packets, got_packet, NULL);

    fprintf(stderr, "\n%d packets captured\n", captured);
    if (ignored > 0) {
	fprintf(stderr, "%d packets ignored (too small to decapsulate)\n",
	    ignored);
	}
    if (got_stats) {
	(void)fprintf(stderr, "%d packets received by filter\n", pcs.ps_recv);
	(void)fprintf(stderr, "%d packets dropped by kernel\n", pcs.ps_drop);

	/*
	 * if packets dropped, check/warn if pcap socket buffer is too small
	 */
	if (pcs.ps_drop > 0) {
	    procf = fopen(RMEM_DEF, "r");
	    if (procf) {fscanf(procf, "%d", &rmem_def); fclose(procf);}
	    procf = fopen(RMEM_MAX, "r");
	    if (procf) {fscanf(procf, "%d", &rmem_max); fclose(procf);}
	    if (rmem_def < RMEM_SUG || rmem_max < RMEM_SUG) {
		fprintf(stderr, "\nNote %s may drop fewer packets "
		    "if you increase:\n  %s and\n  %s\nto %d or more\n\n",
		    progname, RMEM_MAX, RMEM_DEF, RMEM_SUG);
		}
	    }
	}