tcpip.c

harvest是一个下载html网页得机器人

/*
 * Copyright (c) 1995-2003, Index Data
 * See the file LICENSE for details.
 *
 * $Id: tcpip.c,v 1.58 2003/05/20 20:33:29 adam Exp $
 */

#include <stdio.h>
#include <string.h>
#include <stdlib.h>

#ifdef WIN32
#else
#include <unistd.h>
#endif

#include <errno.h>
#include <fcntl.h>
#include <signal.h>
#if HAVE_OPENSSL_SSL_H
#include <openssl/ssl.h>
#include <openssl/err.h>
#endif

#include <yaz/comstack.h>
#include <yaz/tcpip.h>
#include <yaz/log.h>
#include <yaz/nmem.h>

#ifdef WIN32
#else
#include <netinet/tcp.h>
#endif

static int tcpip_close(COMSTACK h);
static int tcpip_put(COMSTACK h, char *buf, int size);
static int tcpip_get(COMSTACK h, char **buf, int *bufsize);
static int tcpip_connect(COMSTACK h, void *address);
static int tcpip_more(COMSTACK h);
static int tcpip_rcvconnect(COMSTACK h);
static int tcpip_bind(COMSTACK h, void *address, int mode);
static int tcpip_listen(COMSTACK h, char *raddr, int *addrlen,
		 int (*check_ip)(void *cd, const char *a, int len, int type),
		 void *cd);
static int tcpip_set_blocking(COMSTACK p, int blocking);

#if HAVE_OPENSSL_SSL_H
static int ssl_get(COMSTACK h, char **buf, int *bufsize);
static int ssl_put(COMSTACK h, char *buf, int size);
#endif

static COMSTACK tcpip_accept(COMSTACK h);
static char *tcpip_addrstr(COMSTACK h);
static void *tcpip_straddr(COMSTACK h, const char *str);

#if 0
#define TRC(x) x
#else
#define TRC(X)
#endif

#ifndef YAZ_SOCKLEN_T
#define YAZ_SOCKLEN_T int
#endif

/* this state is used for both SSL and straight TCP/IP */
typedef struct tcpip_state
{
    char *altbuf; /* alternate buffer for surplus data */
    int altsize;  /* size as xmalloced */
    int altlen;   /* length of data or 0 if none */

    int written;  /* -1 if we aren't writing */
    int towrite;  /* to verify against user input */
    int (*complete)(const unsigned char *buf, int len); /* length/comple. */
    struct sockaddr_in addr;  /* returned by cs_straddr */
    char buf[128]; /* returned by cs_addrstr */
#if HAVE_OPENSSL_SSL_H
    SSL_CTX *ctx;
    SSL_CTX *ctx_alloc;
    SSL *ssl;
#endif
} tcpip_state;

#ifdef WIN32
static int tcpip_init (void)
{
    static int initialized = 0;
    if (!initialized)
    {
        WORD requested;
        WSADATA wd;

        requested = MAKEWORD(1, 1);
        if (WSAStartup(requested, &wd))
            return 0;
        initialized = 1;
    }
    return 1;
}
#else
static int tcpip_init (void)
{
    return 1;
}
#endif

/*
 * This function is always called through the cs_create() macro.
 * s >= 0: socket has already been established for us.
 */
COMSTACK tcpip_type(int s, int blocking, int protocol, void *vp)
{
    COMSTACK p;
    tcpip_state *state;
    int new_socket;
#ifdef WIN32
    unsigned long tru = 1;
#endif

    if (!tcpip_init ())
        return 0;
    if (s < 0)
    {
	if ((s = socket(AF_INET, SOCK_STREAM, 0)) < 0)
	    return 0;
	new_socket = 1;
    }
    else
	new_socket = 0;
    if (!(p = (struct comstack *)xmalloc(sizeof(struct comstack))))
	return 0;
    if (!(state = (struct tcpip_state *)(p->cprivate =
                                         xmalloc(sizeof(tcpip_state)))))
	return 0;

#ifdef WIN32
    if (!(p->blocking = blocking) && ioctlsocket(s, FIONBIO, &tru) < 0)
        return 0;
#else
    if (!(p->blocking = blocking))
    {   
        if (fcntl(s, F_SETFL, O_NONBLOCK) < 0)
            return 0;
#ifndef MSG_NOSIGNAL
        signal (SIGPIPE, SIG_IGN);
#endif
    }
#endif

    p->io_pending = 0;
    p->iofile = s;
    p->type = tcpip_type;
    p->protocol = (enum oid_proto) protocol;

    p->f_connect = tcpip_connect;
    p->f_rcvconnect = tcpip_rcvconnect;
    p->f_get = tcpip_get;
    p->f_put = tcpip_put;
    p->f_close = tcpip_close;
    p->f_more = tcpip_more;
    p->f_bind = tcpip_bind;
    p->f_listen = tcpip_listen;
    p->f_accept = tcpip_accept;
    p->f_addrstr = tcpip_addrstr;
    p->f_straddr = tcpip_straddr;
    p->f_set_blocking = tcpip_set_blocking;

    p->state = new_socket ? CS_ST_UNBND : CS_ST_IDLE; /* state of line */
    p->event = CS_NONE;
    p->cerrno = 0;
    p->stackerr = 0;

#if HAVE_OPENSSL_SSL_H
    state->ctx = state->ctx_alloc = 0;
    state->ssl = 0;
#endif

    state->altbuf = 0;
    state->altsize = state->altlen = 0;
    state->towrite = state->written = -1;
    if (protocol == PROTO_WAIS)
	state->complete = completeWAIS;
    else
	state->complete = cs_complete_auto;

    p->timeout = COMSTACK_DEFAULT_TIMEOUT;
    TRC(fprintf(stderr, "Created new TCPIP comstack\n"));

    return p;
}

#if HAVE_OPENSSL_SSL_H

COMSTACK ssl_type(int s, int blocking, int protocol, void *vp)
{
    tcpip_state *state;
    COMSTACK p;
    yaz_log(LOG_LOG, "ssl_type begin");

    p = tcpip_type (s, blocking, protocol, 0);
    if (!p)
	return 0;
    p->f_get = ssl_get;
    p->f_put = ssl_put;
    p->type = ssl_type;
    state = (tcpip_state *) p->cprivate;
    if (vp)
	state->ctx = vp;
    else
    {
	SSL_load_error_strings();
	SSLeay_add_all_algorithms();

	state->ctx = state->ctx_alloc = SSL_CTX_new (SSLv23_method());
	if (!state->ctx)
	{
	    tcpip_close(p);
	    return 0;
	}
    }
    /* note: we don't handle already opened socket in SSL mode - yet */
    yaz_log(LOG_LOG, "ssl_type end");
    return p;
}
#endif

int tcpip_strtoaddr_ex(const char *str, struct sockaddr_in *add,
                       int default_port)
{
    struct hostent *hp;
    char *p, buf[512];
    short int port = default_port;
    unsigned tmpadd;

    if (!tcpip_init ())
        return 0;
    TRC(fprintf(stderr, "tcpip_strtoaddress: %s\n", str ? str : "NULL"));
    add->sin_family = AF_INET;
    strncpy(buf, str, 511);
    buf[511] = 0;
    if ((p = strchr(buf, '/')))
        *p = 0;
    if ((p = strchr(buf, ':')))
    {
        *p = 0;
        port = atoi(p + 1);
    }
    add->sin_port = htons(port);
    if (!strcmp("@", buf))
        add->sin_addr.s_addr = INADDR_ANY;
    else if ((hp = gethostbyname(buf)))
        memcpy(&add->sin_addr.s_addr, *hp->h_addr_list,
	       sizeof(struct in_addr));
    else if ((tmpadd = (unsigned) inet_addr(buf)) != 0)
        memcpy(&add->sin_addr.s_addr, &tmpadd, sizeof(struct in_addr));
    else
        return 0;
    return 1;
}

void *tcpip_straddr(COMSTACK h, const char *str)
{
    tcpip_state *sp = (tcpip_state *)h->cprivate;
    int port = 210;

    if (h->protocol == PROTO_HTTP)
        port = 80;

    if (!tcpip_strtoaddr_ex (str, &sp->addr, port))
	return 0;
    return &sp->addr;
}

struct sockaddr_in *tcpip_strtoaddr(const char *str)
{
    static struct sockaddr_in add;
    
    if (!tcpip_strtoaddr_ex (str, &add, 210))
	return 0;
    return &add;
}

int tcpip_more(COMSTACK h)
{
    tcpip_state *sp = (tcpip_state *)h->cprivate;
    
    return sp->altlen && (*sp->complete)((unsigned char *) sp->altbuf,
	sp->altlen);
}

/*
 * connect(2) will block (sometimes) - nothing we can do short of doing
 * weird things like spawning subprocesses or threading or some weird junk
 * like that.
 */
int tcpip_connect(COMSTACK h, void *address)
{
    struct sockaddr_in *add = (struct sockaddr_in *)address;
#if HAVE_OPENSSL_SSL_H
    tcpip_state *sp = (tcpip_state *)h->cprivate;
#endif
    int r;
#ifdef __sun__
    int recbuflen;
    YAZ_SOCKLEN_T rbufsize = sizeof(recbuflen);
#endif
    TRC(fprintf(stderr, "tcpip_connect\n"));
    h->io_pending = 0;
    if (h->state != CS_ST_UNBND)
    {
        h->cerrno = CSOUTSTATE;
	return -1;
    }
#ifdef __sun__
    /* On Suns, you must set a bigger Receive Buffer BEFORE a call to connect
     * This gives the connect a chance to negotiate with the other side
     * (see 'man tcp') 
     */
    if ( getsockopt(h->iofile, SOL_SOCKET, SO_RCVBUF, (void *)&recbuflen, &rbufsize ) < 0 )
    {
        h->cerrno = CSYSERR;
        return -1;
    }
    TRC(fprintf( stderr, "Current Size of TCP Receive Buffer= %d\n",
		 recbuflen ));
    recbuflen *= 10; /* lets be optimistic */
    if ( setsockopt(h->iofile, SOL_SOCKET, SO_RCVBUF, (void *)&recbuflen, rbufsize ) < 0 )
    {
        h->cerrno = CSYSERR;
        return -1;
    }
    if ( getsockopt(h->iofile, SOL_SOCKET, SO_RCVBUF, (void *)&recbuflen, &rbufsize ) )
    {
        h->cerrno = CSYSERR;
        return -1;
    }
    TRC(fprintf( stderr, "New Size of TCP Receive Buffer = %d\n",
		 recbuflen ));
#endif
    r = connect(h->iofile, (struct sockaddr *) add, sizeof(*add));
    if (r < 0)
    {
#ifdef WIN32
        if (WSAGetLastError() == WSAEWOULDBLOCK)
        {
            h->event = CS_CONNECT;
            h->state = CS_ST_CONNECTING;
            h->io_pending = CS_WANT_WRITE;
            return 1;
        }
#else
        if (yaz_errno() == EINPROGRESS)
        {
            h->event = CS_CONNECT;
            h->state = CS_ST_CONNECTING;
            h->io_pending = CS_WANT_WRITE|CS_WANT_READ;
            return 1;
        }
#endif
        h->cerrno = CSYSERR;
        return -1;
    }
    h->event = CS_CONNECT;
    h->state = CS_ST_CONNECTING;

    return tcpip_rcvconnect (h);
}

/*
 * nop
 */
int tcpip_rcvconnect(COMSTACK h)
{
#if HAVE_OPENSSL_SSL_H
    tcpip_state *sp = (tcpip_state *)h->cprivate;
#endif
    TRC(fprintf(stderr, "tcpip_rcvconnect\n"));

    if (h->state == CS_ST_DATAXFER)
        return 0;
    if (h->state != CS_ST_CONNECTING)
    {
        h->cerrno = CSOUTSTATE;
	return -1;
    }
#if HAVE_OPENSSL_SSL_H
    if (sp->ctx)
    {
	int res;

	if (!sp->ssl)
	{
	    sp->ssl = SSL_new (sp->ctx);
	    SSL_set_fd (sp->ssl, h->iofile);
	}
	res = SSL_connect (sp->ssl);
	if (res <= 0)
	{
	    int err = SSL_get_error(sp->ssl, res);
	    if (err == SSL_ERROR_WANT_READ)
	    {
		h->io_pending = CS_WANT_READ;
		return 1;
	    }
	    if (err == SSL_ERROR_WANT_WRITE)
	    {
		h->io_pending = CS_WANT_WRITE;
		return 1;
	    }
	    h->cerrno = CSERRORSSL;
	    return -1;
	}
    }
#endif
    h->event = CS_DATA;
    h->state = CS_ST_DATAXFER;
    return 0;
}

#define CERTF "ztest.pem"
#define KEYF "ztest.pem"

static void tcpip_setsockopt (int fd)
{
#if 0
    int len = 4096;
    int set = 1;
    
    if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, (char*)&set, sizeof(int)))
    {
        yaz_log(LOG_WARN|LOG_ERRNO, "setsockopt TCP_NODELAY");
    }
    if (setsockopt(fd, SOL_SOCKET, SO_SNDBUF, (char*)&len, sizeof(int)))
    {
        yaz_log(LOG_WARN|LOG_ERRNO, "setsockopt SNDBUF");
    }
    if (setsockopt(fd, SOL_SOCKET, SO_RCVBUF, (char*)&len, sizeof(int)))
    {
        yaz_log(LOG_WARN|LOG_ERRNO, "setsockopt RCVBUF");
    }
#endif
}

static int tcpip_bind(COMSTACK h, void *address, int mode)
{
    struct sockaddr *addr = (struct sockaddr *)address;
#ifdef WIN32
    BOOL one = 1;
#else
    unsigned long one = 1;
#endif

#if HAVE_OPENSSL_SSL_H
    tcpip_state *sp = (tcpip_state *)h->cprivate;
    if (sp->ctx)
    {
	if (sp->ctx_alloc)
	{
	    int res;
	    res = SSL_CTX_use_certificate_file (sp->ctx, CERTF,
						SSL_FILETYPE_PEM);
	    if (res <= 0)
	    {
		ERR_print_errors_fp(stderr);
		exit (2);
	    }
	    res = SSL_CTX_use_PrivateKey_file (sp->ctx, KEYF,
					       SSL_FILETYPE_PEM);
	    if (res <= 0)
	    {
		ERR_print_errors_fp(stderr);
		exit (3);
	    }
	    res = SSL_CTX_check_private_key (sp->ctx);
	    if (res <= 0)
	    {
		ERR_print_errors_fp(stderr);
		exit(5);
	    }
	}
	TRC (fprintf (stderr, "ssl_bind\n"));
    }
    else
    {
	TRC (fprintf (stderr, "tcpip_bind\n"));
    }
#else
    TRC (fprintf (stderr, "tcpip_bind\n"));
#endif
#ifndef WIN32
    if (setsockopt(h->iofile, SOL_SOCKET, SO_REUSEADDR, (char*) 
	&one, sizeof(one)) < 0)
    {
        h->cerrno = CSYSERR;
        return -1;
    }
#endif
    tcpip_setsockopt(h->iofile);
    if (bind(h->iofile, addr, sizeof(struct sockaddr_in)))
    {
        h->cerrno = CSYSERR;
        return -1;
    }
    if (mode == CS_SERVER && listen(h->iofile, 3) < 0)
    {
        h->cerrno = CSYSERR;
        return -1;
    }
    h->state = CS_ST_IDLE;
    h->event = CS_LISTEN;
    return 0;
}

int tcpip_listen(COMSTACK h, char *raddr, int *addrlen,
		 int (*check_ip)(void *cd, const char *a, int len, int t),
		 void *cd)
{
    struct sockaddr_in addr;
    YAZ_SOCKLEN_T len = sizeof(addr);

    TRC(fprintf(stderr, "tcpip_listen pid=%d\n", getpid()));
    if (h->state != CS_ST_IDLE)
    {
        h->cerrno = CSOUTSTATE;
        return -1;
    }
    h->newfd = accept(h->iofile, (struct sockaddr*)&addr, &len);
    if (h->newfd < 0)
    {
	if (
#ifdef WIN32
	    WSAGetLastError() == WSAEWOULDBLOCK
#else
	    yaz_errno() == EWOULDBLOCK 
#ifdef EAGAIN
#if EAGAIN != EWOULDBLOCK
            || yaz_errno() == EAGAIN
#endif
#endif
#endif
	    )
	    h->cerrno = CSNODATA;
	else
	    h->cerrno = CSYSERR;
        return -1;
    }
    if (addrlen && (size_t) (*addrlen) >= sizeof(struct sockaddr_in))
        memcpy(raddr, &addr, *addrlen = sizeof(struct sockaddr_in));
    else if (addrlen)