seshigh.c

harvest是一个下载html网页得机器人

 */
static Z_NamePlusRecord *surrogatediagrec(association *assoc, char *dbname,
					  int error, char *addinfo)
{
    Z_NamePlusRecord *rec = (Z_NamePlusRecord *)
	odr_malloc (assoc->encode, sizeof(*rec));
    int *err = odr_intdup(assoc->encode, error);
    Z_DiagRec *drec = (Z_DiagRec *)odr_malloc (assoc->encode, sizeof(*drec));
    Z_DefaultDiagFormat *dr = (Z_DefaultDiagFormat *)
	odr_malloc (assoc->encode, sizeof(*dr));
    
    yaz_log(LOG_DEBUG, "SurrogateDiagnotic: %d -- %s", error, addinfo);
    rec->databaseName = dbname;
    rec->which = Z_NamePlusRecord_surrogateDiagnostic;
    rec->u.surrogateDiagnostic = drec;
    drec->which = Z_DiagRec_defaultFormat;
    drec->u.defaultFormat = dr;
    dr->diagnosticSetId =
	yaz_oidval_to_z3950oid (assoc->encode, CLASS_DIAGSET, VAL_BIB1);
    dr->condition = err;
    set_addinfo (dr, addinfo, assoc->encode);

    return rec;
}

/*
 * multiple nonsurrogate diagnostics.
 */
static Z_DiagRecs *diagrecs(association *assoc, int error, char *addinfo)
{
    Z_DiagRecs *recs = (Z_DiagRecs *)odr_malloc (assoc->encode, sizeof(*recs));
    int *err = odr_intdup(assoc->encode, error);
    Z_DiagRec **recp = (Z_DiagRec **)odr_malloc (assoc->encode, sizeof(*recp));
    Z_DiagRec *drec = (Z_DiagRec *)odr_malloc (assoc->encode, sizeof(*drec));
    Z_DefaultDiagFormat *rec = (Z_DefaultDiagFormat *)
	odr_malloc (assoc->encode, sizeof(*rec));

    yaz_log(LOG_DEBUG, "DiagRecs: %d -- %s", error, addinfo ? addinfo : "");

    recs->num_diagRecs = 1;
    recs->diagRecs = recp;
    recp[0] = drec;
    drec->which = Z_DiagRec_defaultFormat;
    drec->u.defaultFormat = rec;

    rec->diagnosticSetId =
	yaz_oidval_to_z3950oid (assoc->encode, CLASS_DIAGSET, VAL_BIB1);
    rec->condition = err;

    rec->which = Z_DefaultDiagFormat_v2Addinfo;
    rec->u.v2Addinfo = odr_strdup (assoc->encode, addinfo ? addinfo : "");
    return recs;
}

static Z_Records *pack_records(association *a, char *setname, int start,
			       int *num, Z_RecordComposition *comp,
			       int *next, int *pres, oid_value format,
			       Z_ReferenceId *referenceId,
			       int *oid)
{
    int recno, total_length = 0, toget = *num, dumped_records = 0;
    Z_Records *records =
	(Z_Records *) odr_malloc (a->encode, sizeof(*records));
    Z_NamePlusRecordList *reclist =
	(Z_NamePlusRecordList *) odr_malloc (a->encode, sizeof(*reclist));
    Z_NamePlusRecord **list =
	(Z_NamePlusRecord **) odr_malloc (a->encode, sizeof(*list) * toget);

    records->which = Z_Records_DBOSD;
    records->u.databaseOrSurDiagnostics = reclist;
    reclist->num_records = 0;
    reclist->records = list;
    *pres = Z_PRES_SUCCESS;
    *num = 0;
    *next = 0;

    yaz_log(LOG_LOG, "Request to pack %d+%d+%s", start, toget, setname);
    yaz_log(LOG_DEBUG, "pms=%d, mrs=%d", a->preferredMessageSize,
    	a->maximumRecordSize);
    for (recno = start; reclist->num_records < toget; recno++)
    {
    	bend_fetch_rr freq;
	Z_NamePlusRecord *thisrec;
	int this_length = 0;
	/*
	 * we get the number of bytes allocated on the stream before any
	 * allocation done by the backend - this should give us a reasonable
	 * idea of the total size of the data so far.
	 */
	total_length = odr_total(a->encode) - dumped_records;
	freq.errcode = 0;
	freq.errstring = 0;
	freq.basename = 0;
	freq.len = 0;
	freq.record = 0;
	freq.last_in_set = 0;
	freq.setname = setname;
	freq.surrogate_flag = 0;
	freq.number = recno;
	freq.comp = comp;
	freq.request_format = format;
	freq.request_format_raw = oid;
	freq.output_format = format;
	freq.output_format_raw = 0;
	freq.stream = a->encode;
	freq.print = a->print;
	freq.referenceId = referenceId;
        freq.schema = 0;
	(*a->init->bend_fetch)(a->backend, &freq);
	/* backend should be able to signal whether error is system-wide
	   or only pertaining to current record */
	if (freq.errcode)
	{
	    if (!freq.surrogate_flag)
	    {
		char s[20];
		*pres = Z_PRES_FAILURE;
		/* for 'present request out of range',
                   set addinfo to record position if not set */
		if (freq.errcode == 13 && freq.errstring == 0)
		{
		    sprintf (s, "%d", recno);
		    freq.errstring = s;
		}
		return diagrec(a, freq.errcode, freq.errstring);
	    }
	    reclist->records[reclist->num_records] =
		surrogatediagrec(a, freq.basename, freq.errcode,
				 freq.errstring);
	    reclist->num_records++;
	    *next = freq.last_in_set ? 0 : recno + 1;
	    continue;
	}
	if (freq.len >= 0)
	    this_length = freq.len;
	else
	    this_length = odr_total(a->encode) - total_length;
	yaz_log(LOG_DEBUG, "  fetched record, len=%d, total=%d",
	    this_length, total_length);
	if (this_length + total_length > a->preferredMessageSize)
	{
	    /* record is small enough, really */
	    if (this_length <= a->preferredMessageSize)
	    {
	    	yaz_log(LOG_DEBUG, "  Dropped last normal-sized record");
		*pres = Z_PRES_PARTIAL_2;
		break;
	    }
	    /* record can only be fetched by itself */
	    if (this_length < a->maximumRecordSize)
	    {
	    	yaz_log(LOG_DEBUG, "  Record > prefmsgsz");
	    	if (toget > 1)
		{
		    yaz_log(LOG_DEBUG, "  Dropped it");
		    reclist->records[reclist->num_records] =
		   	 surrogatediagrec(a, freq.basename, 16, 0);
		    reclist->num_records++;
		    *next = freq.last_in_set ? 0 : recno + 1;
		    dumped_records += this_length;
		    continue;
		}
	    }
	    else /* too big entirely */
	    {
	    	yaz_log(LOG_LOG, "Record > maxrcdsz this=%d max=%d", this_length, a->maximumRecordSize);
		reclist->records[reclist->num_records] =
		    surrogatediagrec(a, freq.basename, 17, 0);
		reclist->num_records++;
		*next = freq.last_in_set ? 0 : recno + 1;
		dumped_records += this_length;
		continue;
	    }
	}

	if (!(thisrec = (Z_NamePlusRecord *)
	      odr_malloc(a->encode, sizeof(*thisrec))))
	    return 0;
	if (!(thisrec->databaseName = (char *)odr_malloc(a->encode,
	    strlen(freq.basename) + 1)))
	    return 0;
	strcpy(thisrec->databaseName, freq.basename);
	thisrec->which = Z_NamePlusRecord_databaseRecord;

	if (freq.output_format_raw)
	{
	    struct oident *ident = oid_getentbyoid(freq.output_format_raw);
	    freq.output_format = ident->value;
	}
	thisrec->u.databaseRecord = z_ext_record(a->encode, freq.output_format,
						 freq.record, freq.len);
	if (!thisrec->u.databaseRecord)
	    return 0;
	reclist->records[reclist->num_records] = thisrec;
	reclist->num_records++;
	*next = freq.last_in_set ? 0 : recno + 1;
    }
    *num = reclist->num_records;
    return records;
}

static Z_APDU *process_searchRequest(association *assoc, request *reqb,
    int *fd)
{
    Z_SearchRequest *req = reqb->apdu_request->u.searchRequest;
    bend_search_rr *bsrr = 
	(bend_search_rr *)nmem_malloc (reqb->request_mem, sizeof(*bsrr));
    
    yaz_log(LOG_LOG, "Got SearchRequest.");
    bsrr->fd = fd;
    bsrr->request = reqb;
    bsrr->association = assoc;
    bsrr->referenceId = req->referenceId;
    save_referenceId (reqb, bsrr->referenceId);

    yaz_log (LOG_LOG, "ResultSet '%s'", req->resultSetName);
    if (req->databaseNames)
    {
	int i;
	for (i = 0; i < req->num_databaseNames; i++)
	    yaz_log (LOG_LOG, "Database '%s'", req->databaseNames[i]);
    }
    yaz_log_zquery(req->query);

    if (assoc->init->bend_search)
    {
	bsrr->setname = req->resultSetName;
	bsrr->replace_set = *req->replaceIndicator;
	bsrr->num_bases = req->num_databaseNames;
	bsrr->basenames = req->databaseNames;
	bsrr->query = req->query;
	bsrr->stream = assoc->encode;
	bsrr->decode = assoc->decode;
	bsrr->print = assoc->print;
	bsrr->errcode = 0;
	bsrr->hits = 0;
	bsrr->errstring = NULL;
        bsrr->search_info = NULL;
	(assoc->init->bend_search)(assoc->backend, bsrr);
	if (!bsrr->request)
	    return 0;
    }
    return response_searchRequest(assoc, reqb, bsrr, fd);
}

int bend_searchresponse(void *handle, bend_search_rr *bsrr) {return 0;}

/*
 * Prepare a searchresponse based on the backend results. We probably want
 * to look at making the fetching of records nonblocking as well, but
 * so far, we'll keep things simple.
 * If bsrt is null, that means we're called in response to a communications
 * event, and we'll have to get the response for ourselves.
 */
static Z_APDU *response_searchRequest(association *assoc, request *reqb,
    bend_search_rr *bsrt, int *fd)
{
    Z_SearchRequest *req = reqb->apdu_request->u.searchRequest;
    Z_APDU *apdu = (Z_APDU *)odr_malloc (assoc->encode, sizeof(*apdu));
    Z_SearchResponse *resp = (Z_SearchResponse *)
	odr_malloc (assoc->encode, sizeof(*resp));
    int *nulint = odr_intdup (assoc->encode, 0);
    bool_t *sr = odr_intdup(assoc->encode, 1);
    int *next = odr_intdup(assoc->encode, 0);
    int *none = odr_intdup(assoc->encode, Z_RES_NONE);

    apdu->which = Z_APDU_searchResponse;
    apdu->u.searchResponse = resp;
    resp->referenceId = req->referenceId;
    resp->additionalSearchInfo = 0;
    resp->otherInfo = 0;
    *fd = -1;
    if (!bsrt && !bend_searchresponse(assoc->backend, bsrt))
    {
    	yaz_log(LOG_FATAL, "Bad result from backend");
    	return 0;
    }
    else if (bsrt->errcode)
    {
	resp->records = diagrec(assoc, bsrt->errcode, bsrt->errstring);
	resp->resultCount = nulint;
	resp->numberOfRecordsReturned = nulint;
	resp->nextResultSetPosition = nulint;
	resp->searchStatus = nulint;
	resp->resultSetStatus = none;
	resp->presentStatus = 0;
    }
    else
    {
    	int *toget = odr_intdup(assoc->encode, 0);
        int *presst = odr_intdup(assoc->encode, 0);
	Z_RecordComposition comp, *compp = 0;

        yaz_log (LOG_LOG, "resultCount: %d", bsrt->hits);

	resp->records = 0;
	resp->resultCount = &bsrt->hits;

	comp.which = Z_RecordComp_simple;
	/* how many records does the user agent want, then? */
	if (bsrt->hits <= *req->smallSetUpperBound)
	{
	    *toget = bsrt->hits;
	    if ((comp.u.simple = req->smallSetElementSetNames))
	        compp = &comp;
	}
	else if (bsrt->hits < *req->largeSetLowerBound)
	{
	    *toget = *req->mediumSetPresentNumber;
	    if (*toget > bsrt->hits)
		*toget = bsrt->hits;
	    if ((comp.u.simple = req->mediumSetElementSetNames))
	        compp = &comp;
	}
	else
	    *toget = 0;

	if (*toget && !resp->records)
	{
	    oident *prefformat;
	    oid_value form;

	    if (!(prefformat = oid_getentbyoid(req->preferredRecordSyntax)))
		form = VAL_NONE;
	    else
	    	form = prefformat->value;
	    resp->records = pack_records(assoc, req->resultSetName, 1,
		toget, compp, next, presst, form, req->referenceId,
					 req->preferredRecordSyntax);
	    if (!resp->records)
		return 0;
	    resp->numberOfRecordsReturned = toget;
	    resp->nextResultSetPosition = next;
	    resp->searchStatus = sr;
	    resp->resultSetStatus = 0;
	    resp->presentStatus = presst;
	}
	else
	{
	    if (*resp->resultCount)
	    	*next = 1;
	    resp->numberOfRecordsReturned = nulint;
	    resp->nextResultSetPosition = next;
	    resp->searchStatus = sr;
	    resp->resultSetStatus = 0;
	    resp->presentStatus = 0;
	}
    }
    resp->additionalSearchInfo = bsrt->search_info;
    return apdu;
}

/*
 * Maybe we got a little over-friendly when we designed bend_fetch to
 * get only one record at a time. Some backends can optimise multiple-record
 * fetches, and at any rate, there is some overhead involved in
 * all that selecting and hopping around. Problem is, of course, that the
 * frontend can't know ahead of time how many records it'll need to
 * fill the negotiated PDU size. Annoying. Segmentation or not, Z/SR
 * is downright lousy as a bulk data transfer protocol.
 *
 * To start with, we'll do the fetching of records from the backend
 * in one operation: To save some trips in and out of the event-handler,
 * and to simplify the interface to pack_records. At any rate, asynch
 * operation is more fun in operations that have an unpredictable execution
 * speed - which is normally more true for search than for present.
 */
static Z_APDU *process_presentRequest(association *assoc, request *reqb,
				      int *fd)
{
    Z_PresentRequest *req = reqb->apdu_request->u.presentRequest;
    oident *prefformat;
    oid_value form;
    Z_APDU *apdu;
    Z_PresentResponse *resp;
    int *next;
    int *num;

    yaz_log(LOG_LOG, "Got PresentRequest.");

    if (!(prefformat = oid_getentbyoid(req->preferredRecordSyntax)))
	form = VAL_NONE;
    else
	form = prefformat->value;
    resp = (Z_PresentResponse *)odr_malloc (assoc->encode, sizeof(*resp));
    resp->records = 0;
    resp->presentStatus = odr_intdup(assoc->encode, 0);
    if (assoc->init->bend_present)
    {
	bend_present_rr *bprr = (bend_present_rr *)
	    nmem_malloc (reqb->request_mem, sizeof(*bprr));
	bprr->setname = req->resultSetId;
	bprr->start = *req->resultSetStartPoint;
	bprr->number = *req->numberOfRecordsRequested;
	bprr->format = form;
	bprr->comp = req->recordComposition;
	bprr->referenceId = req->referenceId;
	bprr->stream = assoc->encode;
	bprr->print = assoc->print;
	bprr->request = reqb;
	bprr->association = assoc;
	bprr->errcode = 0;
	bprr->errstring = NULL;
	(*assoc->init->bend_present)(assoc->backend, bprr);
	
	if (!bprr->request)
	    return 0;
	if (bprr->errcode)
	{
	    resp->records = diagrec(assoc, bprr->errcode, bprr->errstring);
	    *resp->presentStatus = Z_PRES_FAILURE;
	}
    }