seshigh.c

harvest是一个下载html网页得机器人

	iochan_destroy(h);
    }
}

static int process_z_request(association *assoc, request *req, char **msg);

static void assoc_init_reset(association *assoc)
{
    xfree (assoc->init);
    assoc->init = (bend_initrequest *) xmalloc (sizeof(*assoc->init));

    assoc->init->stream = assoc->encode;
    assoc->init->print = assoc->print;
    assoc->init->auth = 0;
    assoc->init->referenceId = 0;
    assoc->init->implementation_version = 0;
    assoc->init->implementation_id = 0;
    assoc->init->implementation_name = 0;
    assoc->init->bend_sort = NULL;
    assoc->init->bend_search = NULL;
    assoc->init->bend_present = NULL;
    assoc->init->bend_esrequest = NULL;
    assoc->init->bend_delete = NULL;
    assoc->init->bend_scan = NULL;
    assoc->init->bend_segment = NULL;
    assoc->init->bend_fetch = NULL;
    assoc->init->bend_explain = NULL;

    assoc->init->charneg_request = NULL;
    assoc->init->charneg_response = NULL;

    assoc->init->decode = assoc->decode;
    assoc->init->peer_name = 
        odr_strdup (assoc->encode, cs_addrstr(assoc->client_link));
}

static int srw_bend_init(association *assoc)
{
    const char *encoding = "UTF-8";
    Z_External *ce;
    bend_initresult *binitres;
    statserv_options_block *cb = statserv_getcontrol();
    
    assoc_init_reset(assoc);

    assoc->maximumRecordSize = 3000000;
    assoc->preferredMessageSize = 3000000;
#if 1
    ce = yaz_set_proposal_charneg(assoc->decode, &encoding, 1, 0, 0, 1);
    assoc->init->charneg_request = ce->u.charNeg3;
#endif
    if (!(binitres = (*cb->bend_init)(assoc->init)))
    {
    	yaz_log(LOG_WARN, "Bad response from backend.");
    	return 0;
    }
    assoc->backend = binitres->handle;
    return 1;
}

static int srw_bend_fetch(association *assoc, int pos,
                          Z_SRW_searchRetrieveRequest *srw_req,
                          Z_SRW_record *record)
{
    bend_fetch_rr rr;
    ODR o = assoc->encode;

    rr.setname = "default";
    rr.number = pos;
    rr.referenceId = 0;
    rr.request_format = VAL_TEXT_XML;
    rr.request_format_raw = yaz_oidval_to_z3950oid(assoc->decode,
                                                   CLASS_TRANSYN,
                                                   VAL_TEXT_XML);
    rr.comp = (Z_RecordComposition *)
	    odr_malloc(assoc->decode, sizeof(*rr.comp));
    rr.comp->which = Z_RecordComp_complex;
    rr.comp->u.complex = (Z_CompSpec *)
	    odr_malloc(assoc->decode, sizeof(Z_CompSpec));
    rr.comp->u.complex->selectAlternativeSyntax = (bool_t *)
        odr_malloc(assoc->encode, sizeof(bool_t));
    *rr.comp->u.complex->selectAlternativeSyntax = 0;    
    rr.comp->u.complex->num_dbSpecific = 0;
    rr.comp->u.complex->dbSpecific = 0;
    rr.comp->u.complex->num_recordSyntax = 0; 
    rr.comp->u.complex->recordSyntax = 0;

    rr.comp->u.complex->generic = (Z_Specification *) 
	    odr_malloc(assoc->decode, sizeof(Z_Specification));
    rr.comp->u.complex->generic->which = Z_Schema_uri;
    rr.comp->u.complex->generic->schema.uri = srw_req->recordSchema;
    rr.comp->u.complex->generic->elementSpec = 0;
    
    rr.stream = assoc->encode;
    rr.print = assoc->print;

    rr.basename = 0;
    rr.len = 0;
    rr.record = 0;
    rr.last_in_set = 0;
    rr.output_format = VAL_TEXT_XML;
    rr.output_format_raw = 0;
    rr.errcode = 0;
    rr.errstring = 0;
    rr.surrogate_flag = 0;
    rr.schema = srw_req->recordSchema;

    if (!assoc->init->bend_fetch)
        return 1;

    (*assoc->init->bend_fetch)(assoc->backend, &rr);

    if (rr.len >= 0)
    {
        record->recordData_buf = rr.record;
        record->recordData_len = rr.len;
        record->recordPosition = odr_intdup(o, pos);
        if (rr.schema)
            record->recordSchema = odr_strdup(o, rr.schema);
        else
            record->recordSchema = 0;
    }
    return rr.errcode;
}

static void srw_bend_search(association *assoc, request *req,
                            Z_SRW_searchRetrieveRequest *srw_req,
                            Z_SRW_searchRetrieveResponse *srw_res)
{
    int srw_error = 0;
    bend_search_rr rr;
    Z_External *ext;
    
    yaz_log(LOG_LOG, "Got SRW SearchRetrieveRequest");
    yaz_log(LOG_DEBUG, "srw_bend_search");
    if (!assoc->init)
    {
        yaz_log(LOG_DEBUG, "srw_bend_init");
        if (!srw_bend_init(assoc))
        {
            srw_error = 3;  /* assume Authentication error */

            srw_res->num_diagnostics = 1;
            srw_res->diagnostics = (Z_SRW_diagnostic *)
                odr_malloc(assoc->encode, sizeof(*srw_res->diagnostics));
            srw_res->diagnostics[0].code = 
                odr_intdup(assoc->encode, srw_error);
            srw_res->diagnostics[0].details = 0;
            return;
        }
    }
    
    rr.setname = "default";
    rr.replace_set = 1;
    rr.num_bases = 1;
    rr.basenames = &srw_req->database;
    rr.referenceId = 0;

    rr.query = (Z_Query *) odr_malloc (assoc->decode, sizeof(*rr.query));

    if (srw_req->query_type == Z_SRW_query_type_cql)
    {
        ext = (Z_External *) odr_malloc(assoc->decode, sizeof(*ext));
        ext->direct_reference = odr_getoidbystr(assoc->decode, 
                                                "1.2.840.10003.16.2");
        ext->indirect_reference = 0;
        ext->descriptor = 0;
        ext->which = Z_External_CQL;
        ext->u.cql = srw_req->query.cql;

        rr.query->which = Z_Query_type_104;
        rr.query->u.type_104 =  ext;
    }
    else if (srw_req->query_type == Z_SRW_query_type_pqf)
    {
        Z_RPNQuery *RPNquery;
        YAZ_PQF_Parser pqf_parser;

        pqf_parser = yaz_pqf_create ();

        RPNquery = yaz_pqf_parse (pqf_parser, assoc->decode,
                                  srw_req->query.pqf);
        if (!RPNquery)
        {
            const char *pqf_msg;
            size_t off;
            int code = yaz_pqf_error (pqf_parser, &pqf_msg, &off);
            yaz_log(LOG_LOG, "%*s^\n", off+4, "");
            yaz_log(LOG_LOG, "Bad PQF: %s (code %d)\n", pqf_msg, code);
            
            srw_error = 10;
        }

        rr.query->which = Z_Query_type_1;
        rr.query->u.type_1 =  RPNquery;

        yaz_pqf_destroy (pqf_parser);
    }
    else
        srw_error = 11;

    if (!srw_error && srw_req->sort_type != Z_SRW_sort_type_none)
        srw_error = 80;

    if (!srw_error && !assoc->init->bend_search)
        srw_error = 1;

    if (srw_error)
    {
        yaz_log(LOG_DEBUG, "srw_bend_search returned SRW error %d", srw_error);
        srw_res->num_diagnostics = 1;
        srw_res->diagnostics = (Z_SRW_diagnostic *)
	    odr_malloc(assoc->encode, sizeof(*srw_res->diagnostics));
        srw_res->diagnostics[0].code = 
            odr_intdup(assoc->encode, srw_error);
        srw_res->diagnostics[0].details = 0;
        return;
    }
    
    rr.stream = assoc->encode;
    rr.decode = assoc->decode;
    rr.print = assoc->print;
    rr.request = req;
    rr.association = assoc;
    rr.fd = 0;
    rr.hits = 0;
    rr.errcode = 0;
    rr.errstring = 0;
    rr.search_info = 0;
    yaz_log_zquery(rr.query);
    (assoc->init->bend_search)(assoc->backend, &rr);
    srw_res->numberOfRecords = odr_intdup(assoc->encode, rr.hits);
    if (rr.errcode)
    {
        yaz_log(LOG_DEBUG, "bend_search returned Bib-1 code %d", rr.errcode);
        srw_res->num_diagnostics = 1;
        srw_res->diagnostics = (Z_SRW_diagnostic *)
	    odr_malloc(assoc->encode, sizeof(*srw_res->diagnostics));
        srw_res->diagnostics[0].code = 
            odr_intdup(assoc->encode, 
                       yaz_diag_bib1_to_srw (rr.errcode));
        srw_res->diagnostics[0].details = rr.errstring;
        yaz_log(LOG_DEBUG, "srw_bend_search returned SRW error %d",
                *srw_res->diagnostics[0].code);
                
    }
    else
    {
        int number = srw_req->maximumRecords ? *srw_req->maximumRecords : 0;
        int start = srw_req->startRecord ? *srw_req->startRecord : 1;

        yaz_log(LOG_LOG, "Request to pack %d+%d out of %d",
                start, number, rr.hits);

        srw_res->numberOfRecords = odr_intdup(assoc->encode, rr.hits);
        if (number > 0)
        {
            int i;

            if (start > rr.hits)
            {
                yaz_log(LOG_LOG, "Request out or range");
            }
            else
            {
                int j = 0;
                int packing = Z_SRW_recordPacking_string;
                if (start + number > rr.hits)
                    number = rr.hits - start + 1;
                if (srw_req->recordPacking && 
                    !strcmp(srw_req->recordPacking, "xml"))
                    packing = Z_SRW_recordPacking_XML;
                srw_res->records = (Z_SRW_record *)
                    odr_malloc(assoc->encode,
                               number * sizeof(*srw_res->records));
                for (i = 0; i<number; i++)
                {
                    int errcode;
                    
                    srw_res->records[j].recordPacking = packing;
                    srw_res->records[j].recordData_buf = 0;
                    yaz_log(LOG_DEBUG, "srw_bend_fetch %d", i+start);
                    errcode = srw_bend_fetch(assoc, i+start, srw_req,
                                             srw_res->records + j);
                    if (errcode)
                    {
                        srw_res->num_diagnostics = 1;
                        srw_res->diagnostics = (Z_SRW_diagnostic *)
                            odr_malloc(assoc->encode, 
                                       sizeof(*srw_res->diagnostics));
                        srw_res->diagnostics[0].code = 
                            odr_intdup(assoc->encode, 
                                       yaz_diag_bib1_to_srw (errcode));
                        srw_res->diagnostics[0].details = rr.errstring;
                        break;
                    }
                    if (srw_res->records[j].recordData_buf)
                        j++;
                }
                srw_res->num_records = j;
                if (!j)
                    srw_res->records = 0;
            }
        }
    }
}


static void srw_bend_explain(association *assoc, request *req,
                             Z_SRW_explainRequest *srw_req,
                             Z_SRW_explainResponse *srw_res)
{
    yaz_log(LOG_LOG, "Got SRW ExplainRequest");
    if (!assoc->init)
    {
        yaz_log(LOG_DEBUG, "srw_bend_init");
        if (!srw_bend_init(assoc))
            return;
    }
    if (assoc->init && assoc->init->bend_explain)
    {
        bend_explain_rr rr;

        rr.stream = assoc->encode;
        rr.decode = assoc->decode;
        rr.print = assoc->print;
        rr.explain_buf = 0;
        (*assoc->init->bend_explain)(assoc->backend, &rr);
        if (rr.explain_buf)
        {
            srw_res->explainData_buf = rr.explain_buf;
            srw_res->explainData_len = strlen(rr.explain_buf);
        }
    }
}

static int hex_digit (int ch)
{
    if (ch >= '0' && ch <= '9')
        return ch - '0';
    else if (ch >= 'a' && ch <= 'f')
        return ch - 'a'+10;
    else if (ch >= 'A' && ch <= 'F')
        return ch - 'A'+10;
    return 0;
}

static char *uri_val(const char *path, const char *name, ODR o)
{
    size_t nlen = strlen(name);
    if (*path != '?')
        return 0;
    path++;
    while (path && *path)
    {
        const char *p1 = strchr(path, '=');
        if (!p1)
            break;
        if ((size_t)(p1 - path) == nlen && !memcmp(path, name, nlen))
        {
            size_t i = 0;
            char *ret;
            
            path = p1 + 1;
            p1 = strchr(path, '&');
            if (!p1)
                p1 = strlen(path) + path;
            ret = odr_malloc(o, p1 - path + 1);
            while (*path && *path != '&')
            {
                if (*path == '+')
                {
                    ret[i++] = ' ';
                    path++;
                }
                else if (*path == '%' && path[1] && path[2])
                {
                    ret[i++] = hex_digit (path[1])*16 + hex_digit (path[2]);
                    path = path + 3;
                }
                else
                    ret[i++] = *path++;
            }
            ret[i] = '\0';
            return ret;
        }
        path = strchr(p1, '&');
        if (path)
            path++;
    }
    return 0;
}

void uri_val_int(const char *path, const char *name, ODR o, int **intp)
{
    const char *v = uri_val(path, name, o);
    if (v)
        *intp = odr_intdup(o, atoi(v));
}

static void process_http_request(association *assoc, request *req)
{
    Z_HTTP_Request *hreq = req->gdu_request->u.HTTP_Request;
    ODR o = assoc->encode;
    Z_GDU *p = 0;
    Z_HTTP_Response *hres = 0;
    int keepalive = 1;

    if (!strcmp(hreq->method, "GET"))
    {
        char *db = "Default";
        const char *p0 = hreq->path, *p1;
#if HAVE_XML2
        int ret = -1;