zoom-c.c

harvest是一个下载html网页得机器人

	if (!rec)
	    break;
    }
    if (i == resultset->count)
	return zoom_complete;

    apdu = zget_APDU(c->odr_out, Z_APDU_presentRequest);
    req = apdu->u.presentRequest;

    resultset->start += i;
    resultset->count -= i;
    *req->resultSetStartPoint = resultset->start + 1;
    *req->numberOfRecordsRequested = resultset->step>0 ?
        resultset->step : resultset->count;
    assert (*req->numberOfRecordsRequested > 0);

    if (syntax && *syntax)
	req->preferredRecordSyntax =
	    yaz_str_to_z3950oid (c->odr_out, CLASS_RECSYN, syntax);

    if (resultset->schema && *resultset->schema)
    {
	Z_RecordComposition *compo = (Z_RecordComposition *)
            odr_malloc (c->odr_out, sizeof(*compo));

        req->recordComposition = compo;
        compo->which = Z_RecordComp_complex;
        compo->u.complex = (Z_CompSpec *)
            odr_malloc(c->odr_out, sizeof(*compo->u.complex));
        compo->u.complex->selectAlternativeSyntax = (bool_t *) 
            odr_malloc(c->odr_out, sizeof(bool_t));
        *compo->u.complex->selectAlternativeSyntax = 0;

        compo->u.complex->generic = (Z_Specification *)
            odr_malloc(c->odr_out, sizeof(*compo->u.complex->generic));

        compo->u.complex->generic->which = Z_Schema_oid;
        compo->u.complex->generic->schema.oid = (Odr_oid *)
            yaz_str_to_z3950oid (c->odr_out, CLASS_SCHEMA, resultset->schema);

        if (!compo->u.complex->generic->schema.oid)
        {
            /* OID wasn't a schema! Try record syntax instead. */

            compo->u.complex->generic->schema.oid = (Odr_oid *)
                yaz_str_to_z3950oid (c->odr_out, CLASS_RECSYN, resultset->schema);
        }
        if (elementSetName && *elementSetName)
        {
            compo->u.complex->generic->elementSpec = (Z_ElementSpec *)
                odr_malloc(c->odr_out, sizeof(Z_ElementSpec));
            compo->u.complex->generic->elementSpec->which =
                Z_ElementSpec_elementSetName;
            compo->u.complex->generic->elementSpec->u.elementSetName =
                odr_strdup (c->odr_out, elementSetName);
        }
        else
            compo->u.complex->generic->elementSpec = 0;
        compo->u.complex->num_dbSpecific = 0;
        compo->u.complex->dbSpecific = 0;
        compo->u.complex->num_recordSyntax = 0;
        compo->u.complex->recordSyntax = 0;
    }
    else if (elementSetName && *elementSetName)
    {
	Z_ElementSetNames *esn = (Z_ElementSetNames *)
            odr_malloc (c->odr_out, sizeof(*esn));
	Z_RecordComposition *compo = (Z_RecordComposition *)
            odr_malloc (c->odr_out, sizeof(*compo));
	
	esn->which = Z_ElementSetNames_generic;
	esn->u.generic = odr_strdup (c->odr_out, elementSetName);
	compo->which = Z_RecordComp_simple;
	compo->u.simple = esn;
	req->recordComposition = compo;
    }
    req->resultSetId = odr_strdup(c->odr_out, resultset->setname);
    return send_APDU (c, apdu);
}

ZOOM_API(ZOOM_scanset)
ZOOM_connection_scan (ZOOM_connection c, const char *start)
{
    ZOOM_scanset scan = (ZOOM_scanset) xmalloc (sizeof(*scan));

    scan->connection = c;
    scan->odr = odr_createmem (ODR_DECODE);
    scan->options = ZOOM_options_create_with_parent (c->options);
    scan->refcount = 1;
    scan->scan_response = 0;

    if ((scan->termListAndStartPoint =
         p_query_scan(scan->odr, PROTO_Z3950, &scan->attributeSet,
                      start)))
    {
        ZOOM_task task = ZOOM_connection_add_task (c, ZOOM_TASK_SCAN);
        task->u.scan.scan = scan;
        
        (scan->refcount)++;
        if (!c->async)
        {
            while (ZOOM_event (1, &c))
                ;
        }
    }
    return scan;
}

ZOOM_API(void)
ZOOM_scanset_destroy (ZOOM_scanset scan)
{
    if (!scan)
        return;
    (scan->refcount)--;
    if (scan->refcount == 0)
    {
        odr_destroy (scan->odr);
        
        ZOOM_options_destroy (scan->options);
        xfree (scan);
    }
}

static zoom_ret send_package (ZOOM_connection c)
{
    ZOOM_Event event;
    if (!c->tasks)
        return zoom_complete;
    assert (c->tasks->which == ZOOM_TASK_PACKAGE);
    
    event = ZOOM_Event_create (ZOOM_EVENT_SEND_APDU);
    ZOOM_connection_put_event (c, event);
    
    return do_write_ex (c, c->tasks->u.package->buf_out,
                        c->tasks->u.package->len_out);
}

static zoom_ret send_scan (ZOOM_connection c)
{
    ZOOM_scanset scan;
    Z_APDU *apdu = zget_APDU(c->odr_out, Z_APDU_scanRequest);
    Z_ScanRequest *req = apdu->u.scanRequest;
    if (!c->tasks)
        return zoom_complete;
    assert (c->tasks->which == ZOOM_TASK_SCAN);
    scan = c->tasks->u.scan.scan;

    req->termListAndStartPoint = scan->termListAndStartPoint;
    req->attributeSet = scan->attributeSet;

    *req->numberOfTermsRequested =
        ZOOM_options_get_int(scan->options, "number", 10);

    req->preferredPositionInResponse =
        odr_intdup (c->odr_out,
                    ZOOM_options_get_int(scan->options, "position", 1));

    req->stepSize =
        odr_intdup (c->odr_out,
                    ZOOM_options_get_int(scan->options, "stepSize", 0));
    
    req->databaseNames = set_DatabaseNames (c, scan->options, 
                                            &req->num_databaseNames);

    return send_APDU (c, apdu);
}

ZOOM_API(size_t)
ZOOM_scanset_size (ZOOM_scanset scan)
{
    if (!scan || !scan->scan_response || !scan->scan_response->entries)
        return 0;
    return scan->scan_response->entries->num_entries;
}

ZOOM_API(const char *)
ZOOM_scanset_term (ZOOM_scanset scan, size_t pos,
                               int *occ, int *len)
{
    const char *term = 0;
    size_t noent = ZOOM_scanset_size (scan);
    Z_ScanResponse *res = scan->scan_response;
    
    *len = 0;
    *occ = 0;
    if (pos >= noent)
        return 0;
    if (res->entries->entries[pos]->which == Z_Entry_termInfo)
    {
        Z_TermInfo *t = res->entries->entries[pos]->u.termInfo;
        
        if (t->term->which == Z_Term_general)
        {
            term = (const char *) t->term->u.general->buf;
            *len = t->term->u.general->len;
        }
        *occ = t->globalOccurrences ? *t->globalOccurrences : 0;
    }
    return term;
}

ZOOM_API(const char *)
ZOOM_scanset_option_get (ZOOM_scanset scan, const char *key)
{
    return ZOOM_options_get (scan->options, key);
}

ZOOM_API(void)
ZOOM_scanset_option_set (ZOOM_scanset scan, const char *key,
                              const char *val)
{
    ZOOM_options_set (scan->options, key, val);
}

static Z_APDU *create_es_package (ZOOM_package p, int type)
{
    const char *str;
    Z_APDU *apdu = zget_APDU(p->odr_out, Z_APDU_extendedServicesRequest);
    Z_ExtendedServicesRequest *req = apdu->u.extendedServicesRequest;
    
    *req->function = Z_ExtendedServicesRequest_create;
    
    str = ZOOM_options_get(p->options, "package-name");
    if (str && *str)
        req->packageName = nmem_strdup (p->odr_out->mem, str);
    
    str = ZOOM_options_get(p->options, "user-id");
    if (str)
        req->userId = nmem_strdup (p->odr_out->mem, str);
    
    req->packageType = yaz_oidval_to_z3950oid(p->odr_out, CLASS_EXTSERV,
                                              type);

    str = ZOOM_options_get(p->options, "function");
    if (str)
    {
        if (!strcmp (str, "create"))
            *req->function = 1;
        if (!strcmp (str, "delete"))
            *req->function = 2;
        if (!strcmp (str, "modify"))
            *req->function = 3;
    }
    return apdu;
}

static const char *ill_array_lookup (void *clientData, const char *idx)
{
    ZOOM_package p = (ZOOM_package) clientData;
    return ZOOM_options_get (p->options, idx+4);
}

static Z_External *encode_ill_request (ZOOM_package p)
{
    ODR out = p->odr_out;
    ILL_Request *req;
    Z_External *r = 0;
    struct ill_get_ctl ctl;
	
    ctl.odr = p->odr_out;
    ctl.clientData = p;
    ctl.f = ill_array_lookup;
	
    req = ill_get_ILLRequest(&ctl, "ill", 0);
	
    if (!ill_Request (out, &req, 0, 0))
    {
        int ill_request_size;
        char *ill_request_buf = odr_getbuf (out, &ill_request_size, 0);
        if (ill_request_buf)
            odr_setbuf (out, ill_request_buf, ill_request_size, 1);
        return 0;
    }
    else
    {
        oident oid;
        int illRequest_size = 0;
        char *illRequest_buf = odr_getbuf (out, &illRequest_size, 0);
		
        oid.proto = PROTO_GENERAL;
        oid.oclass = CLASS_GENERAL;
        oid.value = VAL_ISO_ILL_1;
		
        r = (Z_External *) odr_malloc (out, sizeof(*r));
        r->direct_reference = odr_oiddup(out,oid_getoidbyent(&oid)); 
        r->indirect_reference = 0;
        r->descriptor = 0;
        r->which = Z_External_single;
		
        r->u.single_ASN1_type = (Odr_oct *)
            odr_malloc (out, sizeof(*r->u.single_ASN1_type));
        r->u.single_ASN1_type->buf = (unsigned char*)
            odr_malloc (out, illRequest_size);
        r->u.single_ASN1_type->len = illRequest_size;
        r->u.single_ASN1_type->size = illRequest_size;
        memcpy (r->u.single_ASN1_type->buf, illRequest_buf, illRequest_size);
    }
    return r;
}

static Z_ItemOrder *encode_item_order(ZOOM_package p)
{
    Z_ItemOrder *req = (Z_ItemOrder *) odr_malloc (p->odr_out, sizeof(*req));
    const char *str;
    
    req->which=Z_IOItemOrder_esRequest;
    req->u.esRequest = (Z_IORequest *) 
        odr_malloc(p->odr_out,sizeof(Z_IORequest));

    /* to keep part ... */
    req->u.esRequest->toKeep = (Z_IOOriginPartToKeep *)
	odr_malloc(p->odr_out,sizeof(Z_IOOriginPartToKeep));
    req->u.esRequest->toKeep->supplDescription = 0;
    req->u.esRequest->toKeep->contact = (Z_IOContact *)
        odr_malloc (p->odr_out, sizeof(*req->u.esRequest->toKeep->contact));
	
    str = ZOOM_options_get(p->options, "contact-name");
    req->u.esRequest->toKeep->contact->name = str ?
        nmem_strdup (p->odr_out->mem, str) : 0;
	
    str = ZOOM_options_get(p->options, "contact-phone");
    req->u.esRequest->toKeep->contact->phone = str ?
        nmem_strdup (p->odr_out->mem, str) : 0;
	
    str = ZOOM_options_get(p->options, "contact-email");
    req->u.esRequest->toKeep->contact->email = str ?
        nmem_strdup (p->odr_out->mem, str) : 0;
	
    req->u.esRequest->toKeep->addlBilling = 0;
	
    /* not to keep part ... */
    req->u.esRequest->notToKeep = (Z_IOOriginPartNotToKeep *)
        odr_malloc(p->odr_out,sizeof(Z_IOOriginPartNotToKeep));
	
    str = ZOOM_options_get(p->options, "itemorder-setname");
    if (!str)
        str = "default";

    if (!*str) 
        req->u.esRequest->notToKeep->resultSetItem = 0;
    else
    {
        req->u.esRequest->notToKeep->resultSetItem = (Z_IOResultSetItem *)
           odr_malloc(p->odr_out, sizeof(Z_IOResultSetItem));

        req->u.esRequest->notToKeep->resultSetItem->resultSetId =
           nmem_strdup (p->odr_out->mem, str);
        req->u.esRequest->notToKeep->resultSetItem->item =
            (int *) odr_malloc(p->odr_out, sizeof(int));
	
        str = ZOOM_options_get(p->options, "itemorder-item");
        *req->u.esRequest->notToKeep->resultSetItem->item =
            (str ? atoi(str) : 1);
    }
    req->u.esRequest->notToKeep->itemRequest = encode_ill_request(p);
    
    return req;
}

ZOOM_API(void)
    ZOOM_package_send (ZOOM_package p, const char *type)
{
    Z_APDU *apdu = 0;
    ZOOM_connection c;
    if (!p)
        return;
    c = p->connection;
    odr_reset (p->odr_out);
    xfree (p->buf_out);
    p->buf_out = 0;
    if (!strcmp(type, "itemorder"))
    {
        Z_External *r;
        apdu = create_es_package (p, VAL_ITEMORDER);
        if (apdu)
        {
            r = (Z_External *) odr_malloc (p->odr_out, sizeof(*r));
            
            r->direct_reference =
                yaz_oidval_to_z3950oid(p->odr_out, CLASS_EXTSERV,
                                       VAL_ITEMORDER);
            r->descriptor = 0;
            r->which = Z_External_itemOrder;
            r->indirect_reference = 0;
            r->u.itemOrder = encode_item_order (p);

            apdu->u.extendedServicesRequest->taskSpecificParameters = r;
        }
    }
    if (apdu)
    {
        if (encode_APDU(p->connection, apdu, p->odr_out) == 0)
        {
            char *buf;

            ZOOM_task task = ZOOM_connection_add_task (c, ZOOM_TASK_PACKAGE);
            task->u.package = p;
            buf = odr_getbuf(p->odr_out, &p->len_out, 0);
            p->buf_out = (char *) xmalloc (p->len_out);
            memcpy (p->buf_out, buf, p->len_out);
            
            (p->refcount)++;
            if (!c->async)
            {
                while (ZOOM_event (1, &c))
                    ;
            }
        }
    }
}

ZOOM_API(ZOOM_package)
    ZOOM_connection_package (ZOOM_connection c, ZOOM_options options)
{
    ZOOM_package p = (ZOOM_package) xmalloc (sizeof(*p));

    p->connection = c;
    p->odr_out = odr_createmem (ODR_ENCODE);
    p->options = ZOOM_options_create_with_parent2 (options, c->options);
    p->refcount = 1;
    p->buf_out = 0;
    p->len_out = 0;
    return p;
}

ZOOM_API(void)
    ZOOM_package_destroy(ZOOM_package p)
{
    if (!p)
        return;
    (p->refcount)--;
    if (p->refcount == 0)
    {
        odr_destroy (p->odr_out);
        xfree (p->buf_out);
        
        ZOOM_options_destroy (p->options);
        xfree (p);
    }
}

ZOOM_API(const char *)
ZOOM_package_option_get (ZOOM_package p, const char *key)
{
    return ZOOM_options_get (p->options, key);
}


ZOOM_API(void)
ZOOM_package_option_set (ZOOM_package p, const char *key,
                              const char *val)
{
    ZOOM_options_set (p->options, key, val);
}

static int ZOOM_connection_exec_task (ZOOM_connection c)
{
    ZOOM_task task = c->tasks;
    zoom_ret ret = zoom_complete;

    if (!task)
    {
        yaz_log (LOG_DEBUG, "ZOOM_connection_exec_task task=<null>");
	return 0;
    }
    yaz_log (LOG_DEBUG, "ZOOM_connection_exec_task type=%d run=%d",
             task->which, task->running);
    if (c->error != ZOOM_ERROR_NONE)
    {
        yaz_log (LOG_DEBUG, "remove tasks because of error = %d", c->error);
        ZOOM_connection_remove_tasks (c);
        return 0;
    }
    if (task->running)
    {
        yaz_log (LOG_DEBUG, "task already running");
	return 0;
    }
    task->running = 1;
    ret = zoom_complete;
    if (c->cs || task->which == ZOOM_TASK_CONNECT)
    {
        switch (task->which)
        {
        case ZOOM_TASK_SEARCH:
            if (c->proto == PROTO_HTTP)