ncbi_http_connector.c

ncbi源码

/*
 * ===========================================================================
 * PRODUCTION $Log: ncbi_http_connector.c,v $
 * PRODUCTION Revision 1000.3  2004/02/12 21:52:44  gouriano
 * PRODUCTION PRODUCTION: UPGRADED [CORE_001] Dev-tree R6.63
 * PRODUCTION
 * ===========================================================================
 */

/*  $Id: ncbi_http_connector.c,v 1000.3 2004/02/12 21:52:44 gouriano Exp $
 * ===========================================================================
 *
 *                            PUBLIC DOMAIN NOTICE
 *               National Center for Biotechnology Information
 *
 *  This software/database is a "United States Government Work" under the
 *  terms of the United States Copyright Act.  It was written as part of
 *  the author's official duties as a United States Government employee and
 *  thus cannot be copyrighted.  This software/database is freely available
 *  to the public for use. The National Library of Medicine and the U.S.
 *  Government have not placed any restriction on its use or reproduction.
 *
 *  Although all reasonable efforts have been taken to ensure the accuracy
 *  and reliability of the software and data, the NLM and the U.S.
 *  Government do not and cannot warrant the performance or results that
 *  may be obtained by using this software or data. The NLM and the U.S.
 *  Government disclaim all warranties, express or implied, including
 *  warranties of performance, merchantability or fitness for any particular
 *  purpose.
 *
 *  Please cite the author in any work or product based on this material.
 *
 * ===========================================================================
 *
 * Author:  Anton Lavrentiev, Denis Vakatov
 *
 * File Description:
 *   Implement CONNECTOR for the HTTP-based network connection
 *
 *   See in "ncbi_connector.h" for the detailed specification of the underlying
 *   connector("CONNECTOR", "SConnectorTag") methods and structures.
 *
 */

#include "ncbi_ansi_ext.h"
#include "ncbi_priv.h"
#include <connect/ncbi_http_connector.h>
#include <ctype.h>
#include <stdlib.h>


/***********************************************************************
 *  INTERNAL -- Auxiliary types and static functions
 ***********************************************************************/

/* If the connector is allowed to connect
 */
typedef enum {
    eCC_None,
    eCC_Once,
    eCC_Unlimited
} ECanConnect;


typedef unsigned EBCanConnect;
typedef unsigned TBHCC_Flags;


/* All internal data necessary to perform the (re)connect and I/O
 *
 * The following states are defined:
 *  "sock"  | "read_header" | State description
 * ---------+---------------+--------------------------------------------------
 *   NULL   |  <whatever>   | User "WRITE" mode: accumulate data in buffer
 * non-NULL |   non-zero    | HTTP header is being read
 * non-NULL |     zero      | HTTP body is being read (user "READ" mode)
 * ---------+---------------+--------------------------------------------------
 */
typedef struct {
    SConnNetInfo*        net_info;        /* network configuration parameters*/
    FHttpParseHTTPHeader parse_http_hdr;  /* callback to parse HTTP reply hdr*/
    FHttpAdjustNetInfo   adjust_net_info; /* for on-the-fly net_info adjust  */
    FHttpAdjustCleanup   adjust_cleanup;  /* supplemental user data...       */
    void*                adjust_data;     /* ...and cleanup routine          */

    TBHCC_Flags          flags:10;        /* as passed to constructor        */
    unsigned             reserved:1;
    unsigned             error_header:1;  /* only err.HTTP header on SOME dbg*/
    EBCanConnect         can_connect:2;   /* whether more conns permitted    */
    unsigned             read_header:1;   /* whether reading header          */
    unsigned             shut_down:1;     /* whether shut down for write     */
    unsigned short       failure_count;   /* incr each failure since open    */

    SOCK                 sock;         /* socket;  NULL if not in "READ" mode*/
    const STimeout*      o_timeout;    /* NULL(infinite), dflt or ptr to next*/
    STimeout             oo_timeout;   /* storage for (finite) open timeout  */
    const STimeout*      w_timeout;    /* NULL(infinite), dflt or ptr to next*/
    STimeout             ww_timeout;   /* storage for a (finite) write tmo   */

    BUF                  http;         /* storage for HTTP reply header      */
    BUF                  r_buf;        /* storage to accumulate input data   */
    BUF                  w_buf;        /* storage to accumulate output data  */
    size_t               w_len;        /* pending message body size          */
} SHttpConnector;


/* Try to fix connection parameters (called for an unconnected connector) */
static int/*bool*/ s_Adjust(SHttpConnector* uuu,
                            char**          redirect,
                            int/*bool*/     drop_unread)
{
    assert(!uuu->sock && uuu->can_connect != eCC_None);
    /* we're here because something is going wrong */
    if (++uuu->failure_count >= uuu->net_info->max_try) {
        if (*redirect) {
            free(*redirect);
            *redirect = 0;
        }
        if (!drop_unread && uuu->failure_count > 1) {
            CORE_LOGF(eLOG_Error, ("[HTTP]  Too many failed attempts (%d),"
                                   " giving up", uuu->failure_count));
        }
        uuu->can_connect = eCC_None;
        return 0/*failure*/;
    }
    /* adjust info before yet another connection attempt */
    if (*redirect) {
        int status = ConnNetInfo_ParseURL(uuu->net_info, *redirect);
        free(*redirect);
        *redirect = 0;
        if (!status) {
            CORE_LOG(eLOG_Error, "[HTTP]  Unable to parse redirect");
            uuu->can_connect = eCC_None;
            return 0/*failure*/;
        }
    } else if (!uuu->adjust_net_info ||
               uuu->adjust_net_info(uuu->net_info,
                                    uuu->adjust_data,
                                    uuu->failure_count) == 0) {
        if (!drop_unread && uuu->failure_count > 1) {
            CORE_LOGF(eLOG_Error, ("[HTTP]  Retry attempts (%d) exhausted,"
                                   " giving up", uuu->failure_count));
        }
        uuu->can_connect = eCC_None;
        return 0/*failure*/;
    }

    ConnNetInfo_AdjustForHttpProxy(uuu->net_info);

    if (uuu->net_info->debug_printout)
        ConnNetInfo_Log(uuu->net_info, CORE_GetLOG());
    return 1/*success*/;
}


/* Unconditionally drop the connection; timeout may specify time allowance */
static void s_DropConnection(SHttpConnector* uuu, const STimeout* timeout)
{
    size_t http_size = BUF_Size(uuu->http);
    assert(uuu->sock);
    if (http_size  &&  BUF_Read(uuu->http, 0, http_size) != http_size) {
        CORE_LOG(eLOG_Error, "[HTTP]  Cannot discard HTTP header buffer");
        assert(0);
    }
    SOCK_SetTimeout(uuu->sock, eIO_Close, timeout);
    SOCK_Close(uuu->sock);
    uuu->sock = 0;
}


/* Connect to the HTTP server, specified by uuu->net_info's "port:host".
 * Return eIO_Success only if socket connection has succeeded and uuu->sock
 * is non-zero. If unsuccessful, try to adjust uuu->net_info by s_Adjust(),
 * and then re-try the connection attempt.
 */
static EIO_Status s_Connect(SHttpConnector* uuu, int/*bool*/ drop_unread)
{
    assert(!uuu->sock);
    if (uuu->can_connect == eCC_None) {
        CORE_LOG(eLOG_Error, "[HTTP]  Connector is no longer usable");
        return eIO_Closed;
    }

    /* the re-try loop... */
    for (;;) {
        int/*bool*/ reset_user_header = 0;
        char*       http_user_header = 0;
        char*       null = 0;

        uuu->w_len = BUF_Size(uuu->w_buf);
        if (uuu->net_info->http_user_header)
            http_user_header = strdup(uuu->net_info->http_user_header);
        if (!uuu->net_info->http_user_header == !http_user_header) {
            ConnNetInfo_ExtendUserHeader
                (uuu->net_info, "User-Agent: NCBIHttpConnector"
#ifdef NCBI_CXX_TOOLKIT
                 " (C++ Toolkit)"
#else
                 " (C Toolkit)"
#endif
                 "\r\n");
            reset_user_header = 1;
        }
        /* connect & send HTTP header */
        uuu->sock = URL_Connect
            (uuu->net_info->host,       uuu->net_info->port,
             uuu->net_info->path,       uuu->net_info->args,
             uuu->net_info->req_method, uuu->w_len,
             uuu->o_timeout,            uuu->w_timeout,
             uuu->net_info->http_user_header,
             (int/*bool*/) (uuu->flags & fHCC_UrlEncodeArgs),
             uuu->net_info->debug_printout == eDebugPrintout_Data
             ? eOn : (uuu->net_info->debug_printout == eDebugPrintout_None
                      ? eOff : eDefault));
        if (reset_user_header) {
            ConnNetInfo_SetUserHeader(uuu->net_info, 0);
            uuu->net_info->http_user_header = http_user_header;
        }

        if (uuu->sock) {
            if (!(uuu->flags & fHCC_NoUpread))
                SOCK_SetReadOnWrite(uuu->sock, eOn);
            return eIO_Success;
        }

        /* connection failed, no socket was created */
        if (!s_Adjust(uuu, &null, drop_unread))
            break;
    }

    return eIO_Closed;
}


/* Connect to the server specified by uuu->net_info, then compose and form
 * relevant HTTP header, and flush the accumulated output data(uuu->w_buf)
 * after the HTTP header. If connection/write unsuccessful, retry to reconnect
 * and send the data again until permitted by s_Adjust().
 */
static EIO_Status s_ConnectAndSend(SHttpConnector* uuu,int/*bool*/ drop_unread)
{
    EIO_Status status;

    for (;;) {
        char*  null = 0;

        if (!uuu->sock) {
            if ((status = s_Connect(uuu, drop_unread)) != eIO_Success)
                break;
            assert(uuu->sock);
            uuu->read_header = 1/*true*/;
            uuu->shut_down = 0/*false*/;
        } else
            status = eIO_Success;

        if (uuu->w_len) {
            size_t off = BUF_Size(uuu->w_buf) - uuu->w_len;

            SOCK_SetTimeout(uuu->sock, eIO_Write, uuu->w_timeout);
            do {
                char   buf[4096];
                size_t n_written;
                size_t n_write = BUF_PeekAt(uuu->w_buf, off, buf, sizeof(buf));
                status = SOCK_Write(uuu->sock, buf, n_write,
                                    &n_written, eIO_WritePlain);
                if (status != eIO_Success)
                    break;
                uuu->w_len -= n_written;
                off        += n_written;
            } while (uuu->w_len);
        } else if (!uuu->shut_down)
            status = SOCK_Write(uuu->sock, 0, 0, 0, eIO_WritePlain);

        if (status == eIO_Success) {
            assert(uuu->w_len == 0);
            if (!uuu->shut_down) {
                /* 10/07/03: While this call here is perfectly legal, it could
                 * cause connection severed by a buggy CISCO load-balancer. */
                /* 10/28/03: CISCO's beta patch for their LB shows that the
                 * problem has been fixed; no more 2'30" drops in connections
                 * that shut down for write.  We still leave this commented
                 * out to allow unpatched clients work seamlessly... */ 
                /*SOCK_Shutdown(uuu->sock, eIO_Write);*/
                uuu->shut_down = 1;
            }
            break;
        }

        if (status == eIO_Timeout &&  uuu->w_timeout  &&
            !uuu->w_timeout->sec  &&  !uuu->w_timeout->usec) {
            break;
        }

        CORE_LOGF(eLOG_Error,
                  ("[HTTP]  Error writing body at offset %lu (%s)",
                   (unsigned long) (BUF_Size(uuu->w_buf) - uuu->w_len),
                   IO_StatusStr(status)));

        /* write failed; close and try to use another server */
        SOCK_Abort(uuu->sock);
        s_DropConnection(uuu, 0/*no wait*/);
        if (!s_Adjust(uuu, &null, drop_unread)) {
            status = eIO_Closed;
            break;
        }
    }

    return status;
}


/* Parse HTTP header */
static EIO_Status s_ReadHeader(SHttpConnector* uuu, char** redirect)
{
    int/*bool*/ moved = 0/*false*/;
    int         server_error = 0;
    int         http_status = 0;
    char*       header;
    size_t      size;

    assert(uuu->sock && uuu->read_header);
    *redirect = 0;
    if (uuu->flags & fHCC_KeepHeader) {
        uuu->read_header = 0;
        return eIO_Success;
    }

    /* line by line HTTP header input */
    for (;;) {
        EIO_Status status;

        /* do we have full header yet? */
        size = BUF_Size(uuu->http);
        if (!(header = (char*) malloc(size + 1))) {
            CORE_LOGF(eLOG_Error, ("[HTTP]  Cannot allocate header, %lu bytes",
                                   (unsigned long) size));
            return eIO_Unknown;
        }
        verify(BUF_Peek(uuu->http, header, size) == size);
        header[size] = '\0';
        if (size >= 4  &&  strcmp(&header[size - 4], "\r\n\r\n") == 0)
            break/*full header captured*/;
        free(header);

        status = SOCK_StripToPattern(uuu->sock, "\r\n", 2, &uuu->http, 0);
        if (status != eIO_Success) {
            const STimeout* tmo = SOCK_GetTimeout(uuu->sock, eIO_Read);
            if (tmo && (tmo->sec || tmo->usec)) {
                CORE_LOGF(eLOG_Error, ("[HTTP]  Error reading header (%s)",
                                       IO_StatusStr(status)));
            }
            return status;
        }
    }
    uuu->read_header = 0/*false*/; /* the entire header has been read */
    if (BUF_Read(uuu->http, 0, size) != size) {
        CORE_LOG(eLOG_Error, "[HTTP]  Cannot discard HTTP header buffer");
        assert(0);
    }

    /* HTTP status must come on the first line of the reply */
    if (sscanf(header, " HTTP/%*d.%*d %d ", &http_status) != 1  ||
        http_status < 200  ||  299 < http_status) {
        server_error = http_status;
        if (http_status == 301  ||  http_status == 302)
            moved = 1;
        else if (http_status == 403  ||  http_status == 404)
            uuu->net_info->max_try = 0;
    }

    if ((server_error || !uuu->error_header) &&
        uuu->net_info->debug_printout == eDebugPrintout_Some) {
        /* HTTP header gets printed as part of data logging when
           uuu->net_info->debug_printout == eDebugPrintout_Data. */
        const char* header_header;
        if (!server_error) 
            header_header = "HTTP header";
        else if (moved)
            header_header = "HTTP header (moved)";
        else if (!uuu->net_info->max_try)
            header_header = "HTTP header (unrecoverable error)";
        else
            header_header = "HTTP header (server error, can retry)";
        CORE_DATA(header, size, header_header);
    }

    if (uuu->parse_http_hdr) {
        if (!(*uuu->parse_http_hdr)
            (header, uuu->adjust_data, server_error))
            server_error = 1;
    }

    if (moved) {
        /* parsing "Location" pointer */
        const char k_LocationTag[] = "\nLocation: ";
        char*      location = strstr(header, k_LocationTag);