zoom-c.c

harvest是一个下载html网页得机器人

/*
 * Copyright (c) 2000-2003, Index Data
 * See the file LICENSE for details.
 *
 * $Id: zoom-c.c,v 1.41 2003/08/19 12:32:29 adam Exp $
 *
 * ZOOM layer for C, connections, result sets, queries.
 */
#include <assert.h>
#include <string.h>
#include "zoom-p.h"

#include <yaz/yaz-util.h>
#include <yaz/xmalloc.h>
#include <yaz/otherinfo.h>
#include <yaz/log.h>
#include <yaz/pquery.h>
#include <yaz/marcdisp.h>
#include <yaz/diagbib1.h>
#include <yaz/charneg.h>
#include <yaz/ill.h>
#include <yaz/srw.h>

#if HAVE_SYS_POLL_H
#include <sys/poll.h>
#endif

typedef enum {
    zoom_pending,
    zoom_complete
} zoom_ret;

static zoom_ret ZOOM_connection_send_init (ZOOM_connection c);
static zoom_ret do_write_ex (ZOOM_connection c, char *buf_out, int len_out);

static ZOOM_Event ZOOM_Event_create (int kind)
{
    ZOOM_Event event = (ZOOM_Event) xmalloc (sizeof(*event));
    event->kind = kind;
    event->next = 0;
    event->prev = 0;
    return event;
}

static void ZOOM_Event_destroy (ZOOM_Event event)
{
    xfree (event);
}

static void ZOOM_connection_put_event (ZOOM_connection c, ZOOM_Event event)
{
    if (c->m_queue_back)
    {
	c->m_queue_back->prev = event;
	assert (c->m_queue_front);
    }
    else
    {
	assert (!c->m_queue_front);
	c->m_queue_front = event;
    }
    event->next = c->m_queue_back;
    event->prev = 0;
    c->m_queue_back = event;
}

static ZOOM_Event ZOOM_connection_get_event(ZOOM_connection c)
{
    ZOOM_Event event = c->m_queue_front;
    if (!event)
    {
        c->last_event = ZOOM_EVENT_NONE;
	return 0;
    }
    assert (c->m_queue_back);
    c->m_queue_front = event->prev;
    if (c->m_queue_front)
    {
	assert (c->m_queue_back);
	c->m_queue_front->next = 0;
    }
    else
	c->m_queue_back = 0;
    c->last_event = event->kind;
    return event;
}


static void set_dset_error (ZOOM_connection c, int error,
                            const char *dset,
                            const char *addinfo, const char *addinfo2)
{
    xfree (c->addinfo);
    c->addinfo = 0;
    c->error = error;
    c->diagset = dset;
    if (addinfo && addinfo2)
    {
        c->addinfo = xmalloc(strlen(addinfo) + strlen(addinfo2) + 2);
        strcpy(c->addinfo, addinfo);
        strcat(c->addinfo, addinfo2);
    }
    else if (addinfo)
        c->addinfo = xstrdup(addinfo);
    if (error)
        yaz_log(LOG_DEBUG, "Error %s %s:%d %s %s",
                c->host_port ? c->host_port : "<>", dset, error,
                addinfo ? addinfo : "",
                addinfo2 ? addinfo2 : "");
}

#if HAVE_XML2
static void set_HTTP_error (ZOOM_connection c, int error,
                            const char *addinfo, const char *addinfo2)
{
    set_dset_error(c, error, "HTTP", addinfo, addinfo2);
}
#endif

static void set_ZOOM_error (ZOOM_connection c, int error,
		            const char *addinfo)
{
    set_dset_error(c, error, "ZOOM", addinfo, 0);
}

static void clear_error (ZOOM_connection c)
{

    switch (c->error)
    {
    case ZOOM_ERROR_CONNECT:
    case ZOOM_ERROR_MEMORY:
    case ZOOM_ERROR_DECODE:
    case ZOOM_ERROR_CONNECTION_LOST:
    case ZOOM_ERROR_INIT:
    case ZOOM_ERROR_INTERNAL:
    case ZOOM_ERROR_UNSUPPORTED_PROTOCOL:
        break;
    default:
        set_ZOOM_error(c, ZOOM_ERROR_NONE, 0);
    }
}

ZOOM_task ZOOM_connection_add_task (ZOOM_connection c, int which)
{
    ZOOM_task *taskp = &c->tasks;
    while (*taskp)
	taskp = &(*taskp)->next;
    *taskp = (ZOOM_task) xmalloc (sizeof(**taskp));
    (*taskp)->running = 0;
    (*taskp)->which = which;
    (*taskp)->next = 0;
    clear_error (c);
    return *taskp;
}

ZOOM_task ZOOM_connection_insert_task (ZOOM_connection c, int which)
{
    ZOOM_task task = (ZOOM_task) xmalloc (sizeof(*task));

    task->next = c->tasks;
    c->tasks = task;

    task->running = 0;
    task->which = which;
    clear_error (c);
    return task;
}

void ZOOM_connection_remove_task (ZOOM_connection c)
{
    ZOOM_task task = c->tasks;

    if (task)
    {
	c->tasks = task->next;
	switch (task->which)
	{
	case ZOOM_TASK_SEARCH:

	    ZOOM_resultset_destroy (task->u.search.resultset);
	    break;
	case ZOOM_TASK_RETRIEVE:
	    ZOOM_resultset_destroy (task->u.retrieve.resultset);
	    break;
        case ZOOM_TASK_CONNECT:
            break;
        case ZOOM_TASK_SCAN:
            ZOOM_scanset_destroy (task->u.scan.scan);
            break;
        case ZOOM_TASK_PACKAGE:
            ZOOM_package_destroy (task->u.package);
            break;
	default:
	    assert (0);
	}
	xfree (task);
    }
}

static int ZOOM_connection_exec_task (ZOOM_connection c);

void ZOOM_connection_remove_tasks (ZOOM_connection c)
{
    while (c->tasks)
	ZOOM_connection_remove_task(c);
}

static ZOOM_record record_cache_lookup (ZOOM_resultset r, int pos);

ZOOM_API(ZOOM_connection)
ZOOM_connection_create (ZOOM_options options)
{
    ZOOM_connection c = (ZOOM_connection) xmalloc (sizeof(*c));

    c->proto = PROTO_Z3950;
    c->cs = 0;
    c->mask = 0;
    c->reconnect_ok = 0;
    c->state = STATE_IDLE;
    c->addinfo = 0;
    set_ZOOM_error(c, ZOOM_ERROR_NONE, 0);
    c->buf_in = 0;
    c->len_in = 0;
    c->buf_out = 0;
    c->len_out = 0;
    c->resultsets = 0;

    c->options = ZOOM_options_create_with_parent(options);

    c->host_port = 0;
    c->path = 0;
    c->proxy = 0;
    
    c->charset = c->lang = 0;

    c->cookie_out = 0;
    c->cookie_in = 0;
    c->client_IP = 0;
    c->tasks = 0;

    c->odr_in = odr_createmem (ODR_DECODE);
    c->odr_out = odr_createmem (ODR_ENCODE);

    c->async = 0;
    c->support_named_resultsets = 0;
    c->last_event = ZOOM_EVENT_NONE;

    c->m_queue_front = 0;
    c->m_queue_back = 0;
    return c;
}

/* set database names. Take local databases (if set); otherwise
   take databases given in ZURL (if set); otherwise use Default */
static char **set_DatabaseNames (ZOOM_connection con, ZOOM_options options,
                                 int *num)
{
    char **databaseNames;
    const char *c;
    int no = 2;
    const char *cp = ZOOM_options_get (options, "databaseName");
    
    if (!cp || !*cp)
    {
        if (strncmp (con->host_port, "unix:", 5) == 0)
	    cp = strchr (con->host_port+5, ':');
	else
	    cp = strchr (con->host_port, '/');
	if (cp)
	    cp++;
    }
    if (cp)
    {
	c = cp;
	while ((c = strchr(c, '+')))
	{
	    c++;
	    no++;
	}
    }
    else
	cp = "Default";
    databaseNames = (char**)
        odr_malloc (con->odr_out, no * sizeof(*databaseNames));
    no = 0;
    while (*cp)
    {
	c = strchr (cp, '+');
	if (!c)
	    c = cp + strlen(cp);
	else if (c == cp)
	{
	    cp++;
	    continue;
	}
	/* cp ptr to first char of db name, c is char
	   following db name */
	databaseNames[no] = (char*) odr_malloc (con->odr_out, 1+c-cp);
	memcpy (databaseNames[no], cp, c-cp);
	databaseNames[no++][c-cp] = '\0';
	cp = c;
	if (*cp)
	    cp++;
    }
    databaseNames[no] = NULL;
    *num = no;
    return databaseNames;
}

ZOOM_API(ZOOM_connection)
ZOOM_connection_new (const char *host, int portnum)
{
    ZOOM_connection c = ZOOM_connection_create (0);

    ZOOM_connection_connect (c, host, portnum);
    return c;
}

ZOOM_API(void)
ZOOM_connection_connect(ZOOM_connection c,
                        const char *host, int portnum)
{
    const char *val;
    ZOOM_task task;

    if (c->cs)
    {
        yaz_log (LOG_DEBUG, "reconnect");
        c->reconnect_ok = 1;
        return;
    }
    yaz_log(LOG_DEBUG, "connect");
    xfree (c->proxy);
    val = ZOOM_options_get (c->options, "proxy");
    if (val && *val)
	c->proxy = xstrdup (val);
    else
	c->proxy = 0;

    xfree (c->charset);
    val = ZOOM_options_get (c->options, "charset");
    if (val && *val)
	c->charset = xstrdup (val);
    else
	c->charset = 0;

    xfree (c->lang);
    val = ZOOM_options_get (c->options, "lang");
    if (val && *val)
	c->lang = xstrdup (val);
    else
	c->lang = 0;

    xfree (c->host_port);
    if (portnum)
    {
	char hostn[128];
	sprintf (hostn, "%.80s:%d", host, portnum);
	c->host_port = xstrdup(hostn);
    }
    else
	c->host_port = xstrdup(host);

    ZOOM_options_set(c->options, "host", c->host_port);

    val = ZOOM_options_get (c->options, "cookie");
    if (val && *val)
        c->cookie_out = xstrdup (val);

    val = ZOOM_options_get (c->options, "clientIP");
    if (val && *val)
        c->client_IP = xstrdup (val);

    c->async = ZOOM_options_get_bool (c->options, "async", 0);
 
    set_ZOOM_error(c, ZOOM_ERROR_NONE, 0);

    task = ZOOM_connection_add_task (c, ZOOM_TASK_CONNECT);

    if (!c->async)
    {
	while (ZOOM_event (1, &c))
	    ;
    }
}

ZOOM_API(ZOOM_query)
ZOOM_query_create(void)
{
    ZOOM_query s = (ZOOM_query) xmalloc (sizeof(*s));

    s->refcount = 1;
    s->z_query = 0;
    s->sort_spec = 0;
    s->odr = odr_createmem (ODR_ENCODE);
    s->query_string = 0;

    return s;
}

ZOOM_API(void)
ZOOM_query_destroy(ZOOM_query s)
{
    if (!s)
	return;

    (s->refcount)--;
    yaz_log (LOG_DEBUG, "ZOOM_query_destroy count=%d", s->refcount);
    if (s->refcount == 0)
    {
	odr_destroy (s->odr);
	xfree (s);
    }
}

ZOOM_API(int)
ZOOM_query_prefix(ZOOM_query s, const char *str)
{
    s->query_string = odr_strdup(s->odr, str);
    s->z_query = (Z_Query *) odr_malloc (s->odr, sizeof(*s->z_query));
    s->z_query->which = Z_Query_type_1;
    s->z_query->u.type_1 =  p_query_rpn(s->odr, PROTO_Z3950, str);
    if (!s->z_query->u.type_1)
	return -1;
    return 0;
}

ZOOM_API(int)
ZOOM_query_cql(ZOOM_query s, const char *str)
{
    Z_External *ext;

    s->query_string = odr_strdup(s->odr, str);

    ext = (Z_External *) odr_malloc(s->odr, sizeof(*ext));
    ext->direct_reference = odr_getoidbystr(s->odr, "1.2.840.10003.16.2");
    ext->indirect_reference = 0;
    ext->descriptor = 0;
    ext->which = Z_External_CQL;
    ext->u.cql = s->query_string;
    
    s->z_query = (Z_Query *) odr_malloc (s->odr, sizeof(*s->z_query));
    s->z_query->which = Z_Query_type_104;
    s->z_query->u.type_104 =  ext;

    return 0;
}

ZOOM_API(int)
ZOOM_query_sortby(ZOOM_query s, const char *criteria)
{
    s->sort_spec = yaz_sort_spec (s->odr, criteria);
    if (!s->sort_spec)
	return -1;
    return 0;
}

static zoom_ret do_write(ZOOM_connection c);

ZOOM_API(void)
ZOOM_connection_destroy(ZOOM_connection c)
{
    ZOOM_resultset r;
    if (!c)
	return;
    if (c->cs)
	cs_close (c->cs);
    for (r = c->resultsets; r; r = r->next)
	r->connection = 0;

    xfree (c->buf_in);
    xfree (c->addinfo);
    odr_destroy (c->odr_in);
    odr_destroy (c->odr_out);
    ZOOM_options_destroy (c->options);
    ZOOM_connection_remove_tasks (c);
    xfree (c->host_port);
    xfree (c->path);
    xfree (c->proxy);
    xfree (c->charset);
    xfree (c->lang);
    xfree (c->cookie_out);
    xfree (c->cookie_in);
    xfree (c->client_IP);
    xfree (c);
}