seshigh.c

harvest是一个下载html网页得机器人

/*
 * Copyright (c) 1995-2003, Index Data
 * See the file LICENSE for details.
 *
 * $Id: seshigh.c,v 1.160 2003/07/16 21:02:06 adam Exp $
 */

/*
 * Frontend server logic.
 *
 * This code receives incoming APDUs, and handles client requests by means
 * of the backend API.
 *
 * Some of the code is getting quite involved, compared to simpler servers -
 * primarily because it is asynchronous both in the communication with
 * the user and the backend. We think the complexity will pay off in
 * the form of greater flexibility when more asynchronous facilities
 * are implemented.
 *
 * Memory management has become somewhat involved. In the simple case, where
 * only one PDU is pending at a time, it will simply reuse the same memory,
 * once it has found its working size. When we enable multiple concurrent
 * operations, perhaps even with multiple parallel calls to the backend, it
 * will maintain a pool of buffers for encoding and decoding, trying to
 * minimize memory allocation/deallocation during normal operation.
 *
 */

#include <stdlib.h>
#include <stdio.h>
#include <sys/types.h>
#ifdef WIN32
#include <io.h>
#define S_ISREG(x) (x & _S_IFREG)
#include <process.h>
#include <sys/stat.h>
#else
#include <sys/stat.h>
#include <unistd.h>
#endif
#include <assert.h>
#include <ctype.h>

#include <yaz/yconfig.h>
#include <yaz/xmalloc.h>
#include <yaz/comstack.h>
#include "eventl.h"
#include "session.h"
#include <yaz/proto.h>
#include <yaz/oid.h>
#include <yaz/log.h>
#include <yaz/logrpn.h>
#include <yaz/statserv.h>
#include <yaz/diagbib1.h>
#include <yaz/charneg.h>
#include <yaz/otherinfo.h>
#include <yaz/yaz-util.h>
#include <yaz/pquery.h>

#include <yaz/srw.h>
#include <yaz/backend.h>

static void process_gdu_request(association *assoc, request *req);
static int process_z_request(association *assoc, request *req, char **msg);
void backend_response(IOCHAN i, int event);
static int process_gdu_response(association *assoc, request *req, Z_GDU *res);
static int process_z_response(association *assoc, request *req, Z_APDU *res);
static Z_APDU *process_initRequest(association *assoc, request *reqb);
static Z_APDU *process_searchRequest(association *assoc, request *reqb,
    int *fd);
static Z_APDU *response_searchRequest(association *assoc, request *reqb,
    bend_search_rr *bsrr, int *fd);
static Z_APDU *process_presentRequest(association *assoc, request *reqb,
    int *fd);
static Z_APDU *process_scanRequest(association *assoc, request *reqb, int *fd);
static Z_APDU *process_sortRequest(association *assoc, request *reqb, int *fd);
static void process_close(association *assoc, request *reqb);
void save_referenceId (request *reqb, Z_ReferenceId *refid);
static Z_APDU *process_deleteRequest(association *assoc, request *reqb,
    int *fd);
static Z_APDU *process_segmentRequest (association *assoc, request *reqb);

static FILE *apduf = 0; /* for use in static mode */
static statserv_options_block *control_block = 0;

static Z_APDU *process_ESRequest(association *assoc, request *reqb, int *fd);

/*
 * Create and initialize a new association-handle.
 *  channel  : iochannel for the current line.
 *  link     : communications channel.
 * Returns: 0 or a new association handle.
 */
association *create_association(IOCHAN channel, COMSTACK link)
{
    association *anew;

    if (!control_block)
    	control_block = statserv_getcontrol();
    if (!(anew = (association *)xmalloc(sizeof(*anew))))
    	return 0;
    anew->init = 0;
    anew->version = 0;
    anew->client_chan = channel;
    anew->client_link = link;
    anew->cs_get_mask = 0;
    anew->cs_put_mask = 0;
    anew->cs_accept_mask = 0;
    if (!(anew->decode = odr_createmem(ODR_DECODE)) ||
    	!(anew->encode = odr_createmem(ODR_ENCODE)))
	return 0;
    if (*control_block->apdufile)
    {
    	char filename[256];
	FILE *f;

	strcpy(filename, control_block->apdufile);
	if (!(anew->print = odr_createmem(ODR_PRINT)))
	    return 0;
	if (*control_block->apdufile == '@')
        {
	    odr_setprint(anew->print, yaz_log_file());
	}	
	else if (*control_block->apdufile != '-')
	{
	    strcpy(filename, control_block->apdufile);
	    if (!control_block->dynamic)
	    {
		if (!apduf)
		{
		    if (!(apduf = fopen(filename, "w")))
		    {
			yaz_log(LOG_WARN|LOG_ERRNO, "%s", filename);
			return 0;
		    }
		    setvbuf(apduf, 0, _IONBF, 0);
		}
		f = apduf;
	    }
	    else 
	    {
		sprintf(filename + strlen(filename), ".%d", getpid());
		if (!(f = fopen(filename, "w")))
		{
		    yaz_log(LOG_WARN|LOG_ERRNO, "%s", filename);
		    return 0;
		}
		setvbuf(f, 0, _IONBF, 0);
	    }
	    odr_setprint(anew->print, f);
    	}
    }
    else
    	anew->print = 0;
    anew->input_buffer = 0;
    anew->input_buffer_len = 0;
    anew->backend = 0;
    anew->state = ASSOC_NEW;
    request_initq(&anew->incoming);
    request_initq(&anew->outgoing);
    anew->proto = cs_getproto(link);
    return anew;
}

/*
 * Free association and release resources.
 */
void destroy_association(association *h)
{
    statserv_options_block *cb = statserv_getcontrol();

    xfree(h->init);
    odr_destroy(h->decode);
    odr_destroy(h->encode);
    if (h->print)
	odr_destroy(h->print);
    if (h->input_buffer)
    xfree(h->input_buffer);
    if (h->backend)
    	(*cb->bend_close)(h->backend);
    while (request_deq(&h->incoming));
    while (request_deq(&h->outgoing));
    request_delq(&h->incoming);
    request_delq(&h->outgoing);
    xfree(h);
    xmalloc_trav("session closed");
    if (control_block && control_block->one_shot)
    {
	exit (0);
    }
}

static void do_close_req(association *a, int reason, char *message,
			 request *req)
{
    Z_APDU apdu;
    Z_Close *cls = zget_Close(a->encode);
    
    /* Purge request queue */
    while (request_deq(&a->incoming));
    while (request_deq(&a->outgoing));
    if (a->version >= 3)
    {
	yaz_log(LOG_LOG, "Sending Close PDU, reason=%d, message=%s",
	    reason, message ? message : "none");
	apdu.which = Z_APDU_close;
	apdu.u.close = cls;
	*cls->closeReason = reason;
	cls->diagnosticInformation = message;
	process_z_response(a, req, &apdu);
	iochan_settimeout(a->client_chan, 20);
    }
    else
    {
	yaz_log(LOG_DEBUG, "v2 client. No Close PDU");
	iochan_setevent(a->client_chan, EVENT_TIMEOUT); /* force imm close */
    }
    a->state = ASSOC_DEAD;
}

static void do_close(association *a, int reason, char *message)
{
    do_close_req (a, reason, message, request_get(&a->outgoing));
}

/*
 * This is where PDUs from the client are read and the further
 * processing is initiated. Flow of control moves down through the
 * various process_* functions below, until the encoded result comes back up
 * to the output handler in here.
 * 
 *  h     : the I/O channel that has an outstanding event.
 *  event : the current outstanding event.
 */
void ir_session(IOCHAN h, int event)
{
    int res;
    association *assoc = (association *)iochan_getdata(h);
    COMSTACK conn = assoc->client_link;
    request *req;

    assert(h && conn && assoc);
    if (event == EVENT_TIMEOUT)
    {
	if (assoc->state != ASSOC_UP)
	{
	    yaz_log(LOG_LOG, "Final timeout - closing connection.");
	    cs_close(conn);
	    destroy_association(assoc);
	    iochan_destroy(h);
	}
	else
	{
	    yaz_log(LOG_LOG, "Session idle too long. Sending close.");
	    do_close(assoc, Z_Close_lackOfActivity, 0);
	}
	return;
    }
    if (event & assoc->cs_accept_mask)
    {
	yaz_log (LOG_DEBUG, "ir_session (accept)");
	if (!cs_accept (conn))
	{
	    yaz_log (LOG_LOG, "accept failed");
	    destroy_association(assoc);
	    iochan_destroy(h);
	}
	iochan_clearflag (h, EVENT_OUTPUT|EVENT_OUTPUT);
	if (conn->io_pending) 
	{   /* cs_accept didn't complete */
	    assoc->cs_accept_mask = 
		((conn->io_pending & CS_WANT_WRITE) ? EVENT_OUTPUT : 0) |
		((conn->io_pending & CS_WANT_READ) ? EVENT_INPUT : 0);

	    iochan_setflag (h, assoc->cs_accept_mask);
	}
	else
	{   /* cs_accept completed. Prepare for reading (cs_get) */
	    assoc->cs_accept_mask = 0;
	    assoc->cs_get_mask = EVENT_INPUT;
	    iochan_setflag (h, assoc->cs_get_mask);
	}
	return;
    }
    if ((event & assoc->cs_get_mask) || (event & EVENT_WORK)) /* input */
    {
    	if ((assoc->cs_put_mask & EVENT_INPUT) == 0 && (event & assoc->cs_get_mask))
	{
	    yaz_log(LOG_DEBUG, "ir_session (input)");
	    /* We aren't speaking to this fellow */
	    if (assoc->state == ASSOC_DEAD)
	    {
		yaz_log(LOG_LOG, "Connection closed - end of session");
		cs_close(conn);
		destroy_association(assoc);
		iochan_destroy(h);
		return;
	    }
	    assoc->cs_get_mask = EVENT_INPUT;
	    if ((res = cs_get(conn, &assoc->input_buffer,
		&assoc->input_buffer_len)) <= 0)
	    {
		yaz_log(LOG_LOG, "Connection closed by client");
		cs_close(conn);
		destroy_association(assoc);
		iochan_destroy(h);
		return;
	    }
	    else if (res == 1) /* incomplete read - wait for more  */
	    {
		if (conn->io_pending & CS_WANT_WRITE)
		    assoc->cs_get_mask |= EVENT_OUTPUT;
		iochan_setflag(h, assoc->cs_get_mask);
		return;
	    }
	    if (cs_more(conn)) /* more stuff - call us again later, please */
		iochan_setevent(h, EVENT_INPUT);
	    	
	    /* we got a complete PDU. Let's decode it */
	    yaz_log(LOG_DEBUG, "Got PDU, %d bytes: lead=%02X %02X %02X", res,
			    assoc->input_buffer[0] & 0xff,
			    assoc->input_buffer[1] & 0xff,
			    assoc->input_buffer[2] & 0xff);
	    req = request_get(&assoc->incoming); /* get a new request */
	    odr_reset(assoc->decode);
	    odr_setbuf(assoc->decode, assoc->input_buffer, res, 0);
	    if (!z_GDU(assoc->decode, &req->gdu_request, 0, 0))
	    {
		yaz_log(LOG_LOG, "ODR error on incoming PDU: %s [element %s] "
                        "[near byte %d] ",
			odr_errmsg(odr_geterror(assoc->decode)),
                        odr_getelement(assoc->decode),
			odr_offset(assoc->decode));
                if (assoc->decode->error != OHTTP)
                {
                    yaz_log(LOG_LOG, "PDU dump:");
                    odr_dumpBER(yaz_log_file(), assoc->input_buffer, res);
                    do_close(assoc, Z_Close_protocolError, "Malformed package");
                }
                else
                {
                    Z_GDU *p = z_get_HTTP_Response(assoc->encode, 400);
                    assoc->state = ASSOC_DEAD;
                    process_gdu_response(assoc, req, p);
                }
		return;
	    }
	    req->request_mem = odr_extract_mem(assoc->decode);
	    if (assoc->print && !z_GDU(assoc->print, &req->gdu_request, 0, 0))
	    {
		yaz_log(LOG_WARN, "ODR print error: %s", 
		    odr_errmsg(odr_geterror(assoc->print)));
		odr_reset(assoc->print);
	    }
	    request_enq(&assoc->incoming, req);
	}

	/* can we do something yet? */
	req = request_head(&assoc->incoming);
	if (req->state == REQUEST_IDLE)
	{
	    request_deq(&assoc->incoming);
	    process_gdu_request(assoc, req);
	}
    }
    if (event & assoc->cs_put_mask)
    {
    	request *req = request_head(&assoc->outgoing);

	assoc->cs_put_mask = 0;
	yaz_log(LOG_DEBUG, "ir_session (output)");
        req->state = REQUEST_PENDING;
    	switch (res = cs_put(conn, req->response, req->len_response))
	{
	case -1:
	    yaz_log(LOG_LOG, "Connection closed by client");
	    cs_close(conn);
	    destroy_association(assoc);
	    iochan_destroy(h);
	    break;
	case 0: /* all sent - release the request structure */
	    yaz_log(LOG_DEBUG, "Wrote PDU, %d bytes", req->len_response);
#if 0
	    yaz_log(LOG_DEBUG, "HTTP out:\n%.*s", req->len_response,
                    req->response);
#endif
	    nmem_destroy(req->request_mem);
	    request_deq(&assoc->outgoing);
	    request_release(req);
	    if (!request_head(&assoc->outgoing))
	    {   /* restore mask for cs_get operation ... */
		iochan_clearflag(h, EVENT_OUTPUT|EVENT_INPUT);
		iochan_setflag(h, assoc->cs_get_mask);
                if (assoc->state == ASSOC_DEAD)
                    iochan_setevent(assoc->client_chan, EVENT_TIMEOUT);
	    }
            else
            {
                assoc->cs_put_mask = EVENT_OUTPUT;
            }
	    break;
	default:
	    if (conn->io_pending & CS_WANT_WRITE)
		assoc->cs_put_mask |= EVENT_OUTPUT;
	    if (conn->io_pending & CS_WANT_READ)
		assoc->cs_put_mask |= EVENT_INPUT;
	    iochan_setflag(h, assoc->cs_put_mask);
	}
    }
    if (event & EVENT_EXCEPT)
    {
	yaz_log(LOG_LOG, "ir_session (exception)");
	cs_close(conn);
	destroy_association(assoc);