zoom-c.c

harvest是一个下载html网页得机器人

    	{
            ODR_MASK_SET(ireq->options, Z_Options_negotiationModel);
            
            oi_unit->which = Z_OtherInfo_externallyDefinedInfo;
            oi_unit->information.externallyDefinedInfo =
                yaz_set_proposal_charneg
                (c->odr_out,
                 (const char **)&c->charset, (c->charset) ? 1:0,
                 (const char **)&c->lang, (c->lang) ? 1:0, 1);
    	}
    }
    assert (apdu);
    return send_APDU (c, apdu);
}

#if HAVE_XML2
static zoom_ret send_srw (ZOOM_connection c, Z_SRW_PDU *sr)
{
    char ctype[50];
    Z_SOAP_Handler h[2] = {
        {"http://www.loc.gov/zing/srw/v1.0/", 0, (Z_SOAP_fun) yaz_srw_codec},
        {0, 0, 0}
    };
    ODR o = odr_createmem(ODR_ENCODE);
    int ret;
    Z_SOAP *p = odr_malloc(o, sizeof(*p));
    Z_GDU *gdu;
    ZOOM_Event event;

    gdu = z_get_HTTP_Request(c->odr_out);
    gdu->u.HTTP_Request->path = c->path;

    if (c->host_port)
    {
        const char *cp0 = strstr(c->host_port, "://");
        const char *cp1 = 0;
        if (cp0)
            cp0 = cp0+3;
        else
            cp0 = c->host_port;

        cp1 = strchr(cp0, '/');
        if (!cp1)
            cp1 = cp0+strlen(cp0);

        if (cp0 && cp1)
        {
            char *h = odr_malloc(c->odr_out, cp1 - cp0 + 1);
            memcpy (h, cp0, cp1 - cp0);
            h[cp1-cp0] = '\0';
            z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers,
                              "host", h);
        }
    }

    strcpy(ctype, "text/xml");
    if (c->charset && strlen(c->charset) < 20)
    {
        strcat(ctype, "; charset=");
        strcat(ctype, c->charset);
    }
    z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers,
                      "Content-Type", ctype);
    z_HTTP_header_add(c->odr_out, &gdu->u.HTTP_Request->headers,
                      "SOAPAction", "\"\"");
    p->which = Z_SOAP_generic;
    p->u.generic = odr_malloc(o, sizeof(*p->u.generic));
    p->u.generic->no = 0;
    p->u.generic->ns = 0;
    p->u.generic->p = sr;
    p->ns = "http://schemas.xmlsoap.org/soap/envelope/";

    ret = z_soap_codec_enc(o, &p,
                           &gdu->u.HTTP_Request->content_buf,
                           &gdu->u.HTTP_Request->content_len, h,
                           c->charset);

    if (!z_GDU(c->odr_out, &gdu, 0, 0))
        return zoom_complete;
    c->buf_out = odr_getbuf(c->odr_out, &c->len_out, 0);

    odr_destroy(o);

    event = ZOOM_Event_create (ZOOM_EVENT_SEND_APDU);
    ZOOM_connection_put_event (c, event);
    odr_reset(c->odr_out);
    return do_write (c);
}
#endif

#if HAVE_XML2
static zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c)
{
    int i;
    ZOOM_resultset resultset = 0;
    Z_SRW_PDU *sr = 0;
    const char *recordPacking = 0;

    if (c->error)                  /* don't continue on error */
	return zoom_complete;
    assert (c->tasks);
    if (c->tasks->which == ZOOM_TASK_SEARCH)
    {
        resultset = c->tasks->u.search.resultset;
        resultset->setname = xstrdup ("default");
        ZOOM_options_set (resultset->options, "setname", resultset->setname);
    }
    else if(c->tasks->which == ZOOM_TASK_RETRIEVE)
    {
        resultset = c->tasks->u.retrieve.resultset;

        resultset->start = c->tasks->u.retrieve.start;
        resultset->count = c->tasks->u.retrieve.count;
        
        if (resultset->start >= resultset->size)
            return zoom_complete;
        if (resultset->start + resultset->count > resultset->size)
            resultset->count = resultset->size - resultset->start;

        for (i = 0; i<resultset->count; i++)
        {
            ZOOM_record rec =
                record_cache_lookup (resultset, i + resultset->start);
            if (!rec)
                break;
        }
        if (i == resultset->count)
            return zoom_complete;
    }
    assert(resultset->query);
        
    sr = yaz_srw_get(c->odr_out, Z_SRW_searchRetrieve_request);

    if (resultset->query->z_query->which == Z_Query_type_104
        && resultset->query->z_query->u.type_104->which == Z_External_CQL)
    {

        sr->u.request->query_type = Z_SRW_query_type_cql;
        sr->u.request->query.cql =resultset->query->z_query->u.type_104->u.cql;
    }
    else if (resultset->query->z_query->which == Z_Query_type_1 &&
             resultset->query->z_query->u.type_1)
    {
        sr->u.request->query_type = Z_SRW_query_type_pqf;
        sr->u.request->query.pqf = resultset->query->query_string;
    }
    else
    {
        set_ZOOM_error(c, ZOOM_ERROR_UNSUPPORTED_QUERY, 0);
        return zoom_complete;
    }
    sr->u.request->startRecord = odr_intdup (c->odr_out, resultset->start + 1);
    sr->u.request->maximumRecords = odr_intdup (
        c->odr_out, resultset->step>0 ? resultset->step : resultset->count);
    sr->u.request->recordSchema = resultset->schema;

    recordPacking = ZOOM_resultset_option_get (resultset, "recordPacking");

    if (recordPacking)
        sr->u.request->recordPacking = odr_strdup(c->odr_out, recordPacking);
    
    return send_srw(c, sr);
}
#else
static zoom_ret ZOOM_connection_srw_send_search(ZOOM_connection c)
{
    return zoom_complete;
}
#endif

static zoom_ret ZOOM_connection_send_search (ZOOM_connection c)
{
    ZOOM_resultset r;
    int lslb, ssub, mspn;
    const char *syntax;
    Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_searchRequest);
    Z_SearchRequest *search_req = apdu->u.searchRequest;
    const char *elementSetName;
    const char *smallSetElementSetName;
    const char *mediumSetElementSetName;

    assert (c->tasks);
    assert (c->tasks->which == ZOOM_TASK_SEARCH);

    r = c->tasks->u.search.resultset;

    elementSetName =
	ZOOM_options_get (r->options, "elementSetName");
    smallSetElementSetName  =
	ZOOM_options_get (r->options, "smallSetElementSetName");
    mediumSetElementSetName =
	ZOOM_options_get (r->options, "mediumSetElementSetName");

    if (!smallSetElementSetName)
	smallSetElementSetName = elementSetName;

    if (!mediumSetElementSetName)
	mediumSetElementSetName = elementSetName;

    assert (r);
    assert (r->query);

    /* prepare query for the search request */
    search_req->query = r->query->z_query;

    search_req->databaseNames =
	set_DatabaseNames (c, r->options, &search_req->num_databaseNames);

    /* get syntax (no need to provide unless piggyback is in effect) */
    syntax = ZOOM_options_get (r->options, "preferredRecordSyntax");

    lslb = ZOOM_options_get_int (r->options, "largeSetLowerBound", -1);
    ssub = ZOOM_options_get_int (r->options, "smallSetUpperBound", -1);
    mspn = ZOOM_options_get_int (r->options, "mediumSetPresentNumber", -1);
    if (lslb != -1 && ssub != -1 && mspn != -1)
    {
	/* So're a Z39.50 expert? Let's hope you don't do sort */
	*search_req->largeSetLowerBound = lslb;
	*search_req->smallSetUpperBound = ssub;
	*search_req->mediumSetPresentNumber = mspn;
    }
    else if (r->start == 0 && r->count > 0
	     && r->piggyback && !r->r_sort_spec && !r->schema)
    {
	/* Regular piggyback - do it unless we're going to do sort */
	*search_req->largeSetLowerBound = 2000000000;
	*search_req->smallSetUpperBound = 1;
	*search_req->mediumSetPresentNumber = r->step>0 ? r->step : r->count;
    }
    else
    {
	/* non-piggyback. Need not provide elementsets or syntaxes .. */
	smallSetElementSetName = 0;
	mediumSetElementSetName = 0;
	syntax = 0;
    }
    if (smallSetElementSetName && *smallSetElementSetName)
    {
	Z_ElementSetNames *esn = (Z_ElementSetNames *)
            odr_malloc (c->odr_out, sizeof(*esn));
	
	esn->which = Z_ElementSetNames_generic;
	esn->u.generic = odr_strdup (c->odr_out, smallSetElementSetName);
	search_req->smallSetElementSetNames = esn;
    }
    if (mediumSetElementSetName && *mediumSetElementSetName)
    {
	Z_ElementSetNames *esn = (Z_ElementSetNames *)
            odr_malloc (c->odr_out, sizeof(*esn));
	
	esn->which = Z_ElementSetNames_generic;
	esn->u.generic = odr_strdup (c->odr_out, mediumSetElementSetName);
	search_req->mediumSetElementSetNames = esn;
    }
    if (syntax)
	search_req->preferredRecordSyntax =
	    yaz_str_to_z3950oid (c->odr_out, CLASS_RECSYN, syntax);
    
    if (!r->setname)
    {
        if (c->support_named_resultsets)
        {
            char setname[14];
            int ord;
            /* find the lowest unused ordinal so that we re-use
               result sets on the server. */
            for (ord = 1; ; ord++)
            {
                ZOOM_resultset rp;
                sprintf (setname, "%d", ord);
                for (rp = c->resultsets; rp; rp = rp->next)
                    if (rp->setname && !strcmp (rp->setname, setname))
                        break;
                if (!rp)
                    break;
            }
            r->setname = xstrdup (setname);
            yaz_log (LOG_DEBUG, "allocating set %s", r->setname);
        }
        else
            r->setname = xstrdup ("default");
        ZOOM_options_set (r->options, "setname", r->setname);
    }
    search_req->resultSetName = odr_strdup(c->odr_out, r->setname);
    /* send search request */
    return send_APDU (c, apdu);
}

static void response_diag (ZOOM_connection c, Z_DiagRec *p)
{
    int oclass;
    Z_DefaultDiagFormat *r;
    char *addinfo = 0;
    
    xfree (c->addinfo);
    c->addinfo = 0;
    if (p->which != Z_DiagRec_defaultFormat)
    {
        set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
	return;
    }
    r = p->u.defaultFormat;
    switch (r->which)
    {
    case Z_DefaultDiagFormat_v2Addinfo:
	addinfo = r->u.v2Addinfo;
	break;
    case Z_DefaultDiagFormat_v3Addinfo:
	addinfo = r->u.v3Addinfo;
	break;
    }
    set_dset_error(c, *r->condition,
                   yaz_z3950oid_to_str(r->diagnosticSetId, &oclass),
                   addinfo, 0);
}

ZOOM_API(ZOOM_record)
ZOOM_record_clone (ZOOM_record srec)
{
    char *buf;
    int size;
    ODR odr_enc;
    ZOOM_record nrec;

    odr_enc = odr_createmem(ODR_ENCODE);
    if (!z_NamePlusRecord (odr_enc, &srec->npr, 0, 0))
	return 0;
    buf = odr_getbuf (odr_enc, &size, 0);
    
    nrec = (ZOOM_record) xmalloc (sizeof(*nrec));
    nrec->odr = odr_createmem(ODR_DECODE);
    nrec->wrbuf_marc = 0;
    nrec->wrbuf_iconv = 0;
    nrec->wrbuf_opac = 0;
    odr_setbuf (nrec->odr, buf, size, 0);
    z_NamePlusRecord (nrec->odr, &nrec->npr, 0, 0);
    
    odr_destroy (odr_enc);
    return nrec;
}

ZOOM_API(ZOOM_record)
ZOOM_resultset_record_immediate (ZOOM_resultset s,size_t pos)
{
    return record_cache_lookup (s, pos);
}

ZOOM_API(ZOOM_record)
ZOOM_resultset_record (ZOOM_resultset r, size_t pos)
{
    ZOOM_record rec = ZOOM_resultset_record_immediate(r, pos);

    if (!rec)
    {
        ZOOM_resultset_retrieve (r, 1, pos, 1);
        rec = ZOOM_resultset_record_immediate (r, pos);
    }
    return rec;
}

ZOOM_API(void)
ZOOM_record_destroy (ZOOM_record rec)
{
    if (!rec)
	return;
    if (rec->wrbuf_marc)
	wrbuf_free (rec->wrbuf_marc, 1);
    if (rec->wrbuf_iconv)
	wrbuf_free (rec->wrbuf_iconv, 1);
    if (rec->wrbuf_opac)
	wrbuf_free (rec->wrbuf_opac, 1);
    odr_destroy (rec->odr);
    xfree (rec);
}

static const char *record_iconv_return(ZOOM_record rec, int *len,
				       const char *buf, int sz,
				       const char *record_charset)
{
    char to[40];
    char from[40];
    yaz_iconv_t cd = 0;

    *from = '\0';
    strcpy(to, "UTF-8");
    if (record_charset && *record_charset)
    {
	/* Use "from,to" or just "from" */
	const char *cp =strchr(record_charset, ',');
	int clen = strlen(record_charset);
	if (cp && cp[1])
	{
	    strncpy( to, cp+1, sizeof(to)-1);
	    to[sizeof(to)-1] = '\0';
	    clen = cp - record_charset;
	}
	if (clen > sizeof(from)-1)
	    clen = sizeof(from)-1;
	
	if (clen)
	    strncpy(from, record_charset, clen);
	from[clen] = '\0';
    }

    if (*from && *to && (cd = yaz_iconv_open(to, from)))
    {
	char outbuf[12];
	size_t inbytesleft = sz;
	const char *inp = buf;
	
	if (!rec->wrbuf_iconv)
	    rec->wrbuf_iconv = wrbuf_alloc();

	wrbuf_rewind(rec->wrbuf_iconv);

	while (inbytesleft)
	{
	    size_t outbytesleft = sizeof(outbuf);
	    char *outp = outbuf;
	    size_t r = yaz_iconv (cd, (char**) &inp,
				  &inbytesleft, 
				  &outp, &outbytesleft);
	    if (r == (size_t) (-1))
	    {
		int e = yaz_iconv_error(cd);
		if (e != YAZ_ICONV_E2BIG)
		    break;
	    }
	    wrbuf_write(rec->wrbuf_iconv, outbuf, outp - outbuf);
	}
	wrbuf_puts(rec->wrbuf_iconv, "");
	buf = wrbuf_buf(rec->wrbuf_iconv);
	sz = wrbuf_len(rec->wrbuf_iconv);
	yaz_iconv_close(cd);
    }
    if (len)
	*len = sz;
    return buf;
}

ZOOM_API(const char *)
ZOOM_record_get (ZOOM_record rec, const char *type_spec, int *len)
{
    char type[40];
    char charset[40];
    const char *cp;
    int i;
    Z_NamePlusRecord *npr;
    
    if (len)
    	*len = 0; /* default return */
    	
    if (!rec)
	return 0;
    npr = rec->npr;
    if (!npr)
	return 0;

    cp = type_spec;
    for (i = 0; cp[i] && i < sizeof(type)-1; i++)
    {
	if (cp[i] == ';' || cp[i] == ' ')
	    break;
	type[i] = cp[i];
    }
    type[i] = '\0';
    charset[0] = '\0';
    if (type_spec[i] == ';')
    {
	i++;
	while (type_spec[i] == ' ')
	    i++;
	if (!strncmp(type_spec+i, "charset=", 8))
	{
	    cp = type_spec+i+8;
	    for (i = 0; cp[i] && i < sizeof(charset)-1; i++)
	    {
		if (cp[i] == ';' || cp[i] == ' ')
		    break;
		charset[i] = cp[i];
	    }
	    charset[i] = '\0';
	}
    }	

    if (!strcmp (type, "database"))
    {
    	if (len)