zoom-c.c

harvest是一个下载html网页得机器人

void ZOOM_resultset_addref (ZOOM_resultset r)
{
    if (r)
    {
	(r->refcount)++;
        yaz_log (LOG_DEBUG, "ZOOM_resultset_addref r=%p count=%d",
                 r, r->refcount);
    }
}
ZOOM_resultset ZOOM_resultset_create ()
{
    ZOOM_resultset r = (ZOOM_resultset) xmalloc (sizeof(*r));

    yaz_log (LOG_DEBUG, "ZOOM_resultset_create r = %p", r);
    r->refcount = 1;
    r->size = 0;
    r->odr = odr_createmem (ODR_ENCODE);
    r->start = 0;
    r->piggyback = 1;
    r->setname = 0;
    r->schema = 0;
    r->count = 0;
    r->step = 0;
    r->record_cache = 0;
    r->r_sort_spec = 0;
    r->query = 0;
    r->connection = 0;
    r->next = 0;
    return r;
}

ZOOM_API(ZOOM_resultset)
ZOOM_connection_search_pqf(ZOOM_connection c, const char *q)
{
    ZOOM_resultset r;
    ZOOM_query s = ZOOM_query_create();

    ZOOM_query_prefix (s, q);

    r = ZOOM_connection_search (c, s);
    ZOOM_query_destroy (s);
    return r;
}

ZOOM_API(ZOOM_resultset)
ZOOM_connection_search(ZOOM_connection c, ZOOM_query q)
{
    ZOOM_resultset r = ZOOM_resultset_create ();
    ZOOM_task task;
    const char *cp;

    r->r_sort_spec = q->sort_spec;
    r->query = q;

    r->options = ZOOM_options_create_with_parent(c->options);

    r->start = ZOOM_options_get_int(r->options, "start", 0);
    r->count = ZOOM_options_get_int(r->options, "count", 0);
    r->step = ZOOM_options_get_int(r->options, "step", 0);
    r->piggyback = ZOOM_options_get_bool (r->options, "piggyback", 1);
    cp = ZOOM_options_get (r->options, "setname");
    if (cp)
        r->setname = xstrdup(cp);
    cp = ZOOM_options_get (r->options, "schema");
    if (cp)
        r->schema = xstrdup(cp);
    
    r->connection = c;

    r->next = c->resultsets;
    c->resultsets = r;

    if (c->host_port && c->proto == PROTO_HTTP)
    {
        if (!c->cs)
        {
            yaz_log(LOG_DEBUG, "NO COMSTACK");
            ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
        }
        else
        {
            yaz_log(LOG_DEBUG, "PREPARE FOR RECONNECT");
            c->reconnect_ok = 1;
        }
    }

    task = ZOOM_connection_add_task (c, ZOOM_TASK_SEARCH);
    task->u.search.resultset = r;
    ZOOM_resultset_addref (r);  

    (q->refcount)++;

    if (!c->async)
    {
	while (ZOOM_event (1, &c))
	    ;
    }
    return r;
}

ZOOM_API(void)
ZOOM_resultset_destroy(ZOOM_resultset r)
{
    if (!r)
        return;
    (r->refcount)--;
    yaz_log (LOG_DEBUG, "ZOOM_resultset_destroy r = %p count=%d",
             r, r->refcount);
    if (r->refcount == 0)
    {
        ZOOM_record_cache rc;

        for (rc = r->record_cache; rc; rc = rc->next)
	{
            if (rc->rec.wrbuf_marc)
                wrbuf_free (rc->rec.wrbuf_marc, 1);
            if (rc->rec.wrbuf_iconv)
                wrbuf_free (rc->rec.wrbuf_iconv, 1);
            if (rc->rec.wrbuf_opac)
                wrbuf_free (rc->rec.wrbuf_opac, 1);
	}
	if (r->connection)
	{
	    /* remove ourselves from the resultsets in connection */
	    ZOOM_resultset *rp = &r->connection->resultsets;
	    while (1)
	    {
		assert (*rp);   /* we must be in this list!! */
		if (*rp == r)
		{   /* OK, we're here - take us out of it */
		    *rp = (*rp)->next;
		    break;
		}
		rp = &(*rp)->next;
	    }
	}
	ZOOM_query_destroy (r->query);
	ZOOM_options_destroy (r->options);
	odr_destroy (r->odr);
        xfree (r->setname);
        xfree (r->schema);
	xfree (r);
    }
}

ZOOM_API(size_t)
ZOOM_resultset_size (ZOOM_resultset r)
{
    return r->size;
}

static void do_close (ZOOM_connection c)
{
    if (c->cs)
	cs_close(c->cs);
    c->cs = 0;
    c->mask = 0;
    c->state = STATE_IDLE;
}

static void ZOOM_resultset_retrieve (ZOOM_resultset r,
                                     int force_sync, int start, int count)
{
    ZOOM_task task;
    ZOOM_connection c;
    const char *cp;

    if (!r)
	return;
    c = r->connection;
    if (!c)
	return;

    if (c->host_port && c->proto == PROTO_HTTP)
    {
        if (!c->cs)
        {
            yaz_log(LOG_DEBUG, "NO COMSTACK");
            ZOOM_connection_add_task(c, ZOOM_TASK_CONNECT);
        }
        else
        {
            yaz_log(LOG_DEBUG, "PREPARE FOR RECONNECT");
            c->reconnect_ok = 1;
        }
    }
    task = ZOOM_connection_add_task (c, ZOOM_TASK_RETRIEVE);
    task->u.retrieve.resultset = r;
    task->u.retrieve.start = start;
    task->u.retrieve.count = count;

    cp = ZOOM_options_get (r->options, "schema");
    if (cp)
    {
        if (!r->schema || strcmp(r->schema, cp))
        {
            xfree(r->schema);
            r->schema = xstrdup(cp);
        }
    }

    ZOOM_resultset_addref (r);

    if (!r->connection->async || force_sync)
	while (r->connection && ZOOM_event (1, &r->connection))
	    ;
}

ZOOM_API(void)
ZOOM_resultset_records (ZOOM_resultset r, ZOOM_record *recs,
                        size_t start, size_t count)
{
    int force_present = 0;

    if (!r)
	return ;
    if (count && recs)
        force_present = 1;
    ZOOM_resultset_retrieve (r, force_present, start, count);
    if (force_present)
    {
        size_t i;
        for (i = 0; i< count; i++)
            recs[i] = ZOOM_resultset_record_immediate (r, i+start);
    }
}

static zoom_ret do_connect (ZOOM_connection c)
{
    void *add;
    const char *effective_host;

    if (c->proxy)
	effective_host = c->proxy;
    else
	effective_host = c->host_port;

    yaz_log (LOG_DEBUG, "do_connect host=%s", effective_host);

    assert (!c->cs);
    c->cs = cs_create_host (effective_host, 0, &add);

    if (c->cs && c->cs->protocol == PROTO_HTTP)
    {
#if HAVE_XML2
        const char *path = 0;

        c->proto = PROTO_HTTP;
        cs_get_host_args(c->host_port, &path);
        xfree(c->path);
        c->path = xmalloc(strlen(path)+2);
        c->path[0] = '/';
        strcpy (c->path+1, path);
#else
        set_ZOOM_error(c, ZOOM_ERROR_UNSUPPORTED_PROTOCOL, "SRW");
        do_close(c);
        return zoom_complete;
#endif
    }
    if (c->cs)
    {
        int ret = cs_connect (c->cs, add);
        if (ret == 0)
        {
            ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_CONNECT);
            ZOOM_connection_put_event(c, event);
            if (c->proto == PROTO_Z3950)
                ZOOM_connection_send_init(c);
            else
            {
                /* no init request for SRW .. */
                assert (c->tasks->which == ZOOM_TASK_CONNECT);
                ZOOM_connection_remove_task (c);
                c->mask = 0;
                ZOOM_connection_exec_task (c);
            }
            c->state = STATE_ESTABLISHED;
            return zoom_pending;
        }
        else if (ret > 0)
        {
            c->state = STATE_CONNECTING; 
            c->mask = ZOOM_SELECT_EXCEPT;
            if (c->cs->io_pending & CS_WANT_WRITE)
                c->mask += ZOOM_SELECT_WRITE;
            if (c->cs->io_pending & CS_WANT_READ)
                c->mask += ZOOM_SELECT_READ;
            return zoom_pending;
        }
    }
    c->state = STATE_IDLE;
    set_ZOOM_error(c, ZOOM_ERROR_CONNECT, effective_host);
    return zoom_complete;
}

int z3950_connection_socket(ZOOM_connection c)
{
    if (c->cs)
	return cs_fileno(c->cs);
    return -1;
}

int z3950_connection_mask(ZOOM_connection c)
{
    if (c->cs)
	return c->mask;
    return 0;
}

static void otherInfo_attach (ZOOM_connection c, Z_APDU *a, ODR out)
{
    int i;
    for (i = 0; i<200; i++)
    {
        size_t len;
	Z_OtherInformation **oi;
        char buf[80];
        const char *val;
        const char *cp;
        int oidval;

        sprintf (buf, "otherInfo%d", i);
        val = ZOOM_options_get (c->options, buf);
        if (!val)
            break;
        cp = strchr (val, ':');
        if (!cp)
            continue;
        len = cp - val;
        if (len >= sizeof(buf))
            len = sizeof(buf)-1;
        memcpy (buf, val, len);
        buf[len] = '\0';
        oidval = oid_getvalbyname (buf);
        if (oidval == VAL_NONE)
            continue;
        
	yaz_oi_APDU(a, &oi);
	yaz_oi_set_string_oidval(oi, out, oidval, 1, cp+1);
    }
}

static int encode_APDU(ZOOM_connection c, Z_APDU *a, ODR out)
{
    assert (a);
    if (c->cookie_out)
    {
	Z_OtherInformation **oi;
	yaz_oi_APDU(a, &oi);
	yaz_oi_set_string_oidval(oi, out, VAL_COOKIE, 1, c->cookie_out);
    }
    if (c->client_IP)
    {
	Z_OtherInformation **oi;
	yaz_oi_APDU(a, &oi);
	yaz_oi_set_string_oidval(oi, out, VAL_CLIENT_IP, 1, c->client_IP);
    }
    otherInfo_attach (c, a, out);
    if (!z_APDU(out, &a, 0, 0))
    {
	FILE *outf = fopen("/tmp/apdu.txt", "a");
	if (a && outf)
	{
	    ODR odr_pr = odr_createmem(ODR_PRINT);
	    fprintf (outf, "a=%p\n", a);
	    odr_setprint(odr_pr, outf);
	    z_APDU(odr_pr, &a, 0, 0);
	    odr_destroy(odr_pr);
	}
        yaz_log (LOG_DEBUG, "encoding failed");
        set_ZOOM_error(c, ZOOM_ERROR_ENCODE, 0);
	odr_reset(out);
	return -1;
    }
    
    return 0;
}

static zoom_ret send_APDU (ZOOM_connection c, Z_APDU *a)
{
    ZOOM_Event event;
    assert (a);
    if (encode_APDU(c, a, c->odr_out))
	return zoom_complete;
    yaz_log(LOG_DEBUG, "send APDU type=%d", a->which);
    c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);
    event = ZOOM_Event_create (ZOOM_EVENT_SEND_APDU);
    ZOOM_connection_put_event (c, event);
    odr_reset(c->odr_out);
    return do_write (c);
}

/* returns 1 if PDU was sent OK (still pending )
           0 if PDU was not sent OK (nothing to wait for) 
*/

static zoom_ret ZOOM_connection_send_init (ZOOM_connection c)
{
    const char *impname;
    Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_initRequest);
    Z_InitRequest *ireq = apdu->u.initRequest;
    Z_IdAuthentication *auth = (Z_IdAuthentication *)
        odr_malloc(c->odr_out, sizeof(*auth));
    const char *auth_groupId = ZOOM_options_get (c->options, "group");
    const char *auth_userId = ZOOM_options_get (c->options, "user");
    const char *auth_password = ZOOM_options_get (c->options, "pass");
    
    ODR_MASK_SET(ireq->options, Z_Options_search);
    ODR_MASK_SET(ireq->options, Z_Options_present);
    ODR_MASK_SET(ireq->options, Z_Options_scan);
    ODR_MASK_SET(ireq->options, Z_Options_sort);
    ODR_MASK_SET(ireq->options, Z_Options_extendedServices);
    ODR_MASK_SET(ireq->options, Z_Options_namedResultSets);
    
    ODR_MASK_SET(ireq->protocolVersion, Z_ProtocolVersion_1);
    ODR_MASK_SET(ireq->protocolVersion, Z_ProtocolVersion_2);
    ODR_MASK_SET(ireq->protocolVersion, Z_ProtocolVersion_3);
    
    impname = ZOOM_options_get (c->options, "implementationName");
    ireq->implementationName =
	(char *) odr_malloc (c->odr_out, 15 + (impname ? strlen(impname) : 0));
    strcpy (ireq->implementationName, "");
    if (impname)
    {
	strcat (ireq->implementationName, impname);
	strcat (ireq->implementationName, "/");
    }					       
    strcat (ireq->implementationName, "ZOOM-C/YAZ");
    
    *ireq->maximumRecordSize =
	ZOOM_options_get_int (c->options, "maximumRecordSize", 1024*1024);
    *ireq->preferredMessageSize =
	ZOOM_options_get_int (c->options, "preferredMessageSize", 1024*1024);
    
    if (auth_groupId || auth_password)
    {
	Z_IdPass *pass = (Z_IdPass *) odr_malloc(c->odr_out, sizeof(*pass));
	int i = 0;
	pass->groupId = 0;
	if (auth_groupId && *auth_groupId)
	{
	    pass->groupId = (char *)
                odr_malloc(c->odr_out, strlen(auth_groupId)+1);
	    strcpy(pass->groupId, auth_groupId);
	    i++;
	}
	pass->userId = 0;
	if (auth_userId && *auth_userId)
	{
	    pass->userId = (char *)
                odr_malloc(c->odr_out, strlen(auth_userId)+1);
	    strcpy(pass->userId, auth_userId);
	    i++;
	}
	pass->password = 0;
	if (auth_password && *auth_password)
	{
	    pass->password = (char *)
                odr_malloc(c->odr_out, strlen(auth_password)+1);
	    strcpy(pass->password, auth_password);
	    i++;
	}
	if (i)
	{
	    auth->which = Z_IdAuthentication_idPass;
	    auth->u.idPass = pass;
	    ireq->idAuthentication = auth;
	}
    }
    else if (auth_userId)
    {
	auth->which = Z_IdAuthentication_open;
	auth->u.open = (char *)
            odr_malloc(c->odr_out, strlen(auth_userId)+1);
	strcpy(auth->u.open, auth_userId);
	ireq->idAuthentication = auth;
    }
    if (c->proxy)
	yaz_oi_set_string_oidval(&ireq->otherInfo, c->odr_out,
				 VAL_PROXY, 1, c->host_port);
    if (c->charset||c->lang)
    {
    	Z_OtherInformation **oi;
    	Z_OtherInformationUnit *oi_unit;
    	
    	yaz_oi_APDU(apdu, &oi);
    	
    	if ((oi_unit = yaz_oi_update(oi, c->odr_out, NULL, 0, 0)))