zoom-c.c

harvest是一个下载html网页得机器人

            *len = (npr->databaseName ? strlen(npr->databaseName) : 0);
	return npr->databaseName;
    }
    else if (!strcmp (type, "syntax"))
    {
        const char *desc = 0;	
	if (npr->which == Z_NamePlusRecord_databaseRecord)
	{
	    Z_External *r = (Z_External *) npr->u.databaseRecord;
	    oident *ent = oid_getentbyoid(r->direct_reference);
	    if (ent)
	    	desc = ent->desc;
	}
	if (!desc)
            desc = "none";
    	if (len)
            *len = strlen(desc);
	return desc;
    }
    else if (!strcmp (type, "render") && 
             npr->which == Z_NamePlusRecord_databaseRecord)
    {
        Z_External *r = (Z_External *) npr->u.databaseRecord;
        oident *ent = oid_getentbyoid(r->direct_reference);

	/* render bibliographic record .. */
	if (r->which == Z_External_OPAC)
	{
	    r = r->u.opac->bibliographicRecord;
	    if (!r)
		return 0;
	    ent = oid_getentbyoid(r->direct_reference);
	}
        if (r->which == Z_External_sutrs)
	    return record_iconv_return(rec, len,
				       r->u.sutrs->buf, r->u.sutrs->len,
				       charset);
        else if (r->which == Z_External_octet)
        {
            yaz_marc_t mt;
            switch (ent->value)
            {
            case VAL_SOIF:
            case VAL_HTML:
            case VAL_SUTRS:
                break;
            case VAL_TEXT_XML:
            case VAL_APPLICATION_XML:
                break;
            default:
                if (!rec->wrbuf_marc)
                    rec->wrbuf_marc = wrbuf_alloc();

                mt = yaz_marc_create();
                wrbuf_rewind (rec->wrbuf_marc);
                if (yaz_marc_decode_wrbuf (
                        mt, (const char *) r->u.octet_aligned->buf,
                        r->u.octet_aligned->len,
                        rec->wrbuf_marc) > 0)
                {
                    yaz_marc_destroy(mt);
		    return record_iconv_return(rec, len,
					       wrbuf_buf(rec->wrbuf_marc),
					       wrbuf_len(rec->wrbuf_marc),
					       charset);
                }
                yaz_marc_destroy(mt);
            }
	    return record_iconv_return(rec, len,
				       (const char *) r->u.octet_aligned->buf,
				       r->u.octet_aligned->len,
				       charset);
        }
        else if (r->which == Z_External_grs1)
        {
            if (!rec->wrbuf_marc)
                rec->wrbuf_marc = wrbuf_alloc();
            wrbuf_rewind (rec->wrbuf_marc);
            yaz_display_grs1(rec->wrbuf_marc, r->u.grs1, 0);
	    return record_iconv_return(rec, len,
				       wrbuf_buf(rec->wrbuf_marc),
				       wrbuf_len(rec->wrbuf_marc),
				       charset);
        }
	return 0;
    }
    else if (npr->which == Z_NamePlusRecord_databaseRecord &&
             (!strcmp (type, "xml") || !strcmp(type, "oai")))
    {
        Z_External *r = (Z_External *) npr->u.databaseRecord;
        oident *ent = oid_getentbyoid(r->direct_reference);

	/* render bibliographic record .. */
	if (r->which == Z_External_OPAC)
	{
	    r = r->u.opac->bibliographicRecord;
	    if (!r)
		return 0;
	    ent = oid_getentbyoid(r->direct_reference);
	}
        
        if (r->which == Z_External_sutrs)
	    return record_iconv_return(rec, len,
				       (const char *) r->u.sutrs->buf,
				       r->u.sutrs->len,
				       charset);
        else if (r->which == Z_External_octet)
        {
            yaz_marc_t mt;
            int marc_decode_type = YAZ_MARC_MARCXML;

            if (!strcmp(type, "oai"))
                marc_decode_type = YAZ_MARC_OAIMARC;
            switch (ent->value)
            {
            case VAL_SOIF:
            case VAL_HTML:
            case VAL_SUTRS:
                break;
            case VAL_TEXT_XML:
            case VAL_APPLICATION_XML:
                break;
            default:
                if (!rec->wrbuf_marc)
                    rec->wrbuf_marc = wrbuf_alloc();
                wrbuf_rewind (rec->wrbuf_marc);
                mt = yaz_marc_create();

                yaz_marc_xml(mt, YAZ_MARC_MARCXML);
                if (yaz_marc_decode_wrbuf (
                        mt, (const char *) r->u.octet_aligned->buf,
                        r->u.octet_aligned->len,
                        rec->wrbuf_marc) > 0)
                {
                    yaz_marc_destroy(mt);
		    return record_iconv_return(rec, len,
					       wrbuf_buf(rec->wrbuf_marc),
					       wrbuf_len(rec->wrbuf_marc),
					       charset);
                }
                yaz_marc_destroy(mt);
            }
	    return record_iconv_return(rec, len,
				       (const char *) r->u.octet_aligned->buf,
				       r->u.octet_aligned->len,
				       charset);
        }
        else if (r->which == Z_External_grs1)
        {
            if (len) *len = 5;
            return "GRS-1";
        }
	return 0;
    }
    else if (!strcmp (type, "raw"))
    {
	if (npr->which == Z_NamePlusRecord_databaseRecord)
	{
	    Z_External *r = (Z_External *) npr->u.databaseRecord;
	    
	    if (r->which == Z_External_sutrs)
	    {
		if (len) *len = r->u.sutrs->len;
		return (const char *) r->u.sutrs->buf;
	    }
	    else if (r->which == Z_External_octet)
	    {
		if (len) *len = r->u.octet_aligned->len;
		return (const char *) r->u.octet_aligned->buf;
	    }
	    else /* grs-1, explain, OPAC, ... */
	    {
		if (len) *len = -1;
                return (const char *) npr->u.databaseRecord;
	    }
	}
	return 0;
    }
    else if (!strcmp (type, "ext"))
    {
	if (npr->which == Z_NamePlusRecord_databaseRecord)
            return (const char *) npr->u.databaseRecord;
	return 0;
    }
    else if (npr->which == Z_NamePlusRecord_databaseRecord &&
             !strcmp (type, "opac"))
	     
    {
	Z_External *r = (Z_External *) npr->u.databaseRecord;
	if (r->which == Z_External_OPAC)
	{
	    if (!rec->wrbuf_opac)
		rec->wrbuf_opac = wrbuf_alloc();
	    wrbuf_rewind (rec->wrbuf_opac);
	    yaz_display_OPAC(rec->wrbuf_opac, r->u.opac, 0);
	    return record_iconv_return(rec, len,
				       wrbuf_buf(rec->wrbuf_opac),
				       wrbuf_len(rec->wrbuf_opac),
				       charset);
	}
    }
    return 0;
}

static int strcmp_null(const char *v1, const char *v2)
{
    if (!v1 && !v2)
        return 0;
    if (!v1 || !v2)
        return -1;
    return strcmp(v1, v2);
}

static void record_cache_add (ZOOM_resultset r, Z_NamePlusRecord *npr, 
                              int pos)
{
    ZOOM_record_cache rc;
    const char *elementSetName =
        ZOOM_resultset_option_get (r, "elementSetName");
    const char *syntax = 
        ZOOM_resultset_option_get (r, "preferredRecordSyntax");
    
    ZOOM_Event event = ZOOM_Event_create(ZOOM_EVENT_RECV_RECORD);
    ZOOM_connection_put_event(r->connection, event);

    for (rc = r->record_cache; rc; rc = rc->next)
    {
	if (pos == rc->pos)
	{
	    if (strcmp_null(r->schema, rc->schema))
                continue;
	    if (strcmp_null(elementSetName,rc->elementSetName))
                continue;
            if (strcmp_null(syntax, rc->syntax))
                continue;
            /* not destroying rc->npr (it's handled by nmem )*/
            rc->rec.npr = npr;
            /* keeping wrbuf_marc too */
            return;
	}
    }
    rc = (ZOOM_record_cache) odr_malloc (r->odr, sizeof(*rc));
    rc->rec.npr = npr; 
    rc->rec.odr = 0;
    rc->rec.wrbuf_marc = 0;
    rc->rec.wrbuf_iconv = 0;
    rc->rec.wrbuf_opac = 0;
    if (elementSetName)
	rc->elementSetName = odr_strdup (r->odr, elementSetName);
    else
	rc->elementSetName = 0;

    if (syntax)
	rc->syntax = odr_strdup (r->odr, syntax);
    else
	rc->syntax = 0;

    if (r->schema)
	rc->schema = odr_strdup (r->odr, r->schema);
    else
	rc->schema = 0;

    rc->pos = pos;
    rc->next = r->record_cache;
    r->record_cache = rc;
}

static ZOOM_record record_cache_lookup (ZOOM_resultset r, int pos)
{
    ZOOM_record_cache rc;
    const char *elementSetName =
        ZOOM_resultset_option_get (r, "elementSetName");
    const char *syntax = 
        ZOOM_resultset_option_get (r, "preferredRecordSyntax");
    
    for (rc = r->record_cache; rc; rc = rc->next)
    {
	if (pos == rc->pos)
	{
	    if (strcmp_null(r->schema, rc->schema))
                continue;
	    if (strcmp_null(elementSetName,rc->elementSetName))
                continue;
            if (strcmp_null(syntax, rc->syntax))
                continue;
            return &rc->rec;
	}
    }
    return 0;
}
					     
static void handle_records (ZOOM_connection c, Z_Records *sr,
			    int present_phase)
{
    ZOOM_resultset resultset;

    if (!c->tasks)
	return ;
    switch (c->tasks->which)
    {
    case ZOOM_TASK_SEARCH:
        resultset = c->tasks->u.search.resultset;
        break;
    case ZOOM_TASK_RETRIEVE:
        resultset = c->tasks->u.retrieve.resultset;        
	break;
    default:
        return;
    }
    if (sr && sr->which == Z_Records_NSD)
    {
	Z_DiagRec dr, *dr_p = &dr;
	dr.which = Z_DiagRec_defaultFormat;
	dr.u.defaultFormat = sr->u.nonSurrogateDiagnostic;
	
	response_diag (c, dr_p);
    }
    else if (sr && sr->which == Z_Records_multipleNSD)
    {
	if (sr->u.multipleNonSurDiagnostics->num_diagRecs >= 1)
	    response_diag(c, sr->u.multipleNonSurDiagnostics->diagRecs[0]);
	else
            set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
    }
    else 
    {
	if (resultset->count + resultset->start > resultset->size)
	    resultset->count = resultset->size - resultset->start;
	if (resultset->count < 0)
	    resultset->count = 0;
	if (sr && sr->which == Z_Records_DBOSD)
	{
	    int i;
	    NMEM nmem = odr_extract_mem (c->odr_in);
	    Z_NamePlusRecordList *p =
		sr->u.databaseOrSurDiagnostics;
	    for (i = 0; i<p->num_records; i++)
	    {
		record_cache_add (resultset, p->records[i],
                                  i+ resultset->start);
	    }
	    /* transfer our response to search_nmem .. we need it later */
	    nmem_transfer (resultset->odr->mem, nmem);
	    nmem_destroy (nmem);
	    if (present_phase && p->num_records == 0)
	    {
		/* present response and we didn't get any records! */
                set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
	    }
	}
	else if (present_phase)
	{
	    /* present response and we didn't get any records! */
            set_ZOOM_error(c, ZOOM_ERROR_DECODE, 0);
	}
    }
}

static void handle_present_response (ZOOM_connection c, Z_PresentResponse *pr)
{
    handle_records (c, pr->records, 1);
}

static void handle_search_response (ZOOM_connection c, Z_SearchResponse *sr)
{
    ZOOM_resultset resultset;
    ZOOM_Event event;
    
    yaz_log (LOG_DEBUG, "got search response");
    
    if (!c->tasks || c->tasks->which != ZOOM_TASK_SEARCH)
	return ;
    
    event = ZOOM_Event_create(ZOOM_EVENT_RECV_SEARCH);
    ZOOM_connection_put_event(c, event);

    resultset = c->tasks->u.search.resultset;

    resultset->size = *sr->resultCount;
    handle_records (c, sr->records, 0);
}

static void sort_response (ZOOM_connection c, Z_SortResponse *res)
{
    if (res->diagnostics && res->num_diagnostics > 0)
	response_diag (c, res->diagnostics[0]);
}

static int scan_response (ZOOM_connection c, Z_ScanResponse *res)
{
    NMEM nmem = odr_extract_mem (c->odr_in);
    ZOOM_scanset scan;

    if (!c->tasks || c->tasks->which != ZOOM_TASK_SCAN)
        return 0;
    scan = c->tasks->u.scan.scan;

    if (res->entries && res->entries->nonsurrogateDiagnostics)
        response_diag(c, res->entries->nonsurrogateDiagnostics[0]);
    scan->scan_response = res;
    nmem_transfer (scan->odr->mem, nmem);
    if (res->stepSize)
        ZOOM_options_set_int (scan->options, "stepSize", *res->stepSize);
    if (res->positionOfTerm)
        ZOOM_options_set_int (scan->options, "position", *res->positionOfTerm);
    if (res->scanStatus)
        ZOOM_options_set_int (scan->options, "scanStatus", *res->scanStatus);
    if (res->numberOfEntriesReturned)
        ZOOM_options_set_int (scan->options, "number",
                              *res->numberOfEntriesReturned);
    nmem_destroy (nmem);
    return 1;
}

static zoom_ret send_sort (ZOOM_connection c)
{
    ZOOM_resultset  resultset;

    if (!c->tasks || c->tasks->which != ZOOM_TASK_SEARCH)
	return zoom_complete;

    resultset = c->tasks->u.search.resultset;

    if (c->error)
    {
	resultset->r_sort_spec = 0;
	return zoom_complete;
    }
    if (resultset->r_sort_spec)
    {
	Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_sortRequest);
	Z_SortRequest *req = apdu->u.sortRequest;
	
	req->num_inputResultSetNames = 1;
	req->inputResultSetNames = (Z_InternationalString **)
	    odr_malloc (c->odr_out, sizeof(*req->inputResultSetNames));
	req->inputResultSetNames[0] =
            odr_strdup (c->odr_out, resultset->setname);
	req->sortedResultSetName = odr_strdup (c->odr_out, resultset->setname);
	req->sortSequence = resultset->r_sort_spec;
	resultset->r_sort_spec = 0;
	return send_APDU (c, apdu);
    }
    return zoom_complete;
}

static zoom_ret send_present (ZOOM_connection c)
{
    Z_APDU *apdu = 0;
    Z_PresentRequest *req = 0;
    int i = 0;
    const char *syntax = 0;
    const char *elementSetName = 0;
    ZOOM_resultset  resultset;

    if (!c->tasks)
	return zoom_complete;

    switch (c->tasks->which)
    {
    case ZOOM_TASK_SEARCH:
        resultset = c->tasks->u.search.resultset;
        break;
    case ZOOM_TASK_RETRIEVE:
        resultset = c->tasks->u.retrieve.resultset;
        resultset->start = c->tasks->u.retrieve.start;
        resultset->count = c->tasks->u.retrieve.count;

        if (resultset->start >= resultset->size)
            return zoom_complete;
        if (resultset->start + resultset->count > resultset->size)
            resultset->count = resultset->size - resultset->start;
	break;
    default:
        return zoom_complete;
    }

    syntax = ZOOM_resultset_option_get (resultset, "preferredRecordSyntax");
    elementSetName = ZOOM_resultset_option_get (resultset, "elementSetName");

    if (c->error)                  /* don't continue on error */
	return zoom_complete;
    if (resultset->start < 0)
	return zoom_complete;
    for (i = 0; i<resultset->count; i++)
    {
	ZOOM_record rec =
	    record_cache_lookup (resultset, i + resultset->start);