waisquery.c

harvest是一个下载html网页得机器人

static char rcsid[] = "waisquery.c,v 1.17 1995/09/06 20:51:54 duane Exp";
/*
 *  waisquery.c - Code for "in-line" Harvest Broker queries using a
 *              WAIS, Inc. WAISserver and the z39.50v2 protocol.
 * 
 *  Darren Hardy, [e-mail], May 1995
 */
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <unistd.h>

/* -------------------------------------------------------------------------- */
/* Based on demo-client.c from WAIS, Inc's client-toolkit. */
/* -------------------------------------------------------------------------- */
/*****************************************************************************
*  (c) Copyright 1994 Wide Area Information Servers, Inc                     *
*      All rights reserved.                                                  *
*                                                                            *
*  This code can be freely copied and used for any purpose as long as        *
*  this notice and copyright notice is included without modification         *
*  at the top of the file.                                                   *
*                                                                            *
*****************************************************************************/
/*---------------------------------------------------------------------------
  ---------------------------------------------------------------------------
 Module:   demo-main.c

 Purpose:  The client-main module provides an example program of how to use
           the client protocol toolkit library.  It makes use of the
           Application Programmers Interface (API) described by the following
           header files:
                         nConnMgr.h
                         appConnMgr.h
                         appConnCli.h
---------------------------------------------------------------------------*/
#include <appConnCli.h>

#define BUFFER_SIZE (256 * 1024)	/* need a big buffer */
#define CLIENT_DESC \
	"Harvest Broker (inline Z39.50-1988 query) v1.3"

/* USE_CONN_PER_QUERY - forces one connection per query to server */
#ifndef USE_CONN_PER_QUERY
#define USE_CONN_PER_QUERY
#endif

/* 
 *  Operations to perform a search:
 *      do_wais_inline_search   - wrapper for doing a search
 *      teardown_search         - 1 time teardown
 *      setup_search            - 1 time initialization (local)
 *      perform_search          - do a search (local)
 */
appConn *conn = NULL;
nConn *net = NULL;
char *theHost = NULL;
int thePort = -1;
char *theDBname = NULL;
char theBuffer[BUFFER_SIZE];


/*---------------------------------------------------------------------------
  ---------------------------------------------------------------------------
 Function:   init
 Purpose:    Initializes the application connection, given by the appConn,
             conn.
 Called by:  main
 Parameters: conn   -- input; a handle to an appConn. It holds
                       information about the state of the connection;
                       may be modified by this function.
             client -- input; a null-terminated string whose value is
                       the client performing the request.
             diags  -- output; a handle to a list of diagnostics,
                       where each item in the list is of type 
                       appConnDiag (see appConnMgr.c).  If the
                       operation was unsuccessful, a list of diags
                       are returned in this parameter.
 Returns:    boolean, true(1) if successful, false(0) otherwise.
---------------------------------------------------------------------------*/

static boolean
 init(appConn * conn,
      char *client,
      list * diags)
{
	long pduType = appConnUnknown;
	appConnStatus as;

	appConnInitReqWrite(conn, client, "", "", NULL, diags);

	if (*diags != NULL)
		return (false);

	while (((as = appConnPduInfo(conn, &pduType, NULL, diags))
		== appConnStatusNotFinished) && (*diags == NULL));

	if (*diags != NULL)
		return (false);

	if (pduType != appConnInitResp)
		return (false);

	while (((as = appConnInitRespRead(conn, NULL, diags))
		== appConnStatusNotFinished) && (*diags == NULL));

	if (*diags != NULL)
		return (false);

	return (true);
}



/*---------------------------------------------------------------------------
  ---------------------------------------------------------------------------
 Function:   search
 Purpose:    Performs a search of a query on a database using the application
             connection, conn.
 Called by:  Demo functions: testSearchAndRetrieveByRecId
                             testSearchWithRFAndRetrieveByPosition
 Parameters: conn         -- input; a handle to an appConn. It holds
                             information about the state of the connection;
                             may be modified by this function.
             maxHits      -- input; a long integer whose value is the
                             maximum number of results the server is 
                             requested to generate. maxHits results are
                             to be stored on the server for the duration
                             of this search.
             qry          -- input; a handle to an appConnQry data type
                             containing the user's query and is created
                             using the appQryAddTerm() function.
             qryTermsUsed -- output; a handle to a list of null-terminated
                             character strings.  Each character string
                             contains the query terms used by the server
                             in the search; these terms may or may not
                             have been specified in the original query.
                             The terms are dependent on the behavior of the
                             server and the contents of the database.  
                             Stop words may have been eliminated, right-
                             truncated words may have been expanded, and
                             synonyms may have been included.
             hits         -- output; a handle to a list.  Each item
                             in the list is of type appConnHeadline, and
                             represents one element of the result set.  
                             The number of hits returned may be less than
                             or equal to the number of hits requested.
             diags        -- output; a handle to a list of diagnostics,
                             where each item in the list is of type 
                             appConnDiag (see appConnMgr.c).  If the
                             operation was unsuccessful, a list of diags
                             are returned in this parameter.
 Returns:    boolean, true(1) if successful, false(0) otherwise.
---------------------------------------------------------------------------*/
static boolean
 search(appConn * conn,
	long maxHits,
	appConnQry * qry,
	list * qryTermsUsed,
	list * hits,
	list * diags)
{
	long pduType = appConnUnknown;
	appConnStatus as;
	long totalNumHits, currentNumHits, requestedNumHits;

	/* Send the search request to the remote system. */
	appConnSearchReqWrite(conn, maxHits, maxHits, rankByServerDefault,
			      theDBname, qry, NULL, diags);

	if (*diags != NULL)
		return (false);

	/* Receive the response. */
	while (((as = appConnPduInfo(conn, &pduType, NULL, diags))
		== appConnStatusNotFinished) && (*diags == NULL));

	if (*diags != NULL)
		return (false);

	/* Make sure the response has the correct packet type. */
	if (pduType != appConnSearchResp)
		return (false);

	/* Read the response and get the headliness. */
	while (((as = appConnSearchRespRead(conn, &totalNumHits, qryTermsUsed,
					    hits, NULL, diags))
		== appConnStatusNotFinished) && (*diags == NULL));

	if (*diags != NULL)
		return (false);

	/* Remember how many headlines we currently have,
	   and how many we want to receive. */
	currentNumHits = length(*hits);
	maxHits = (maxHits < 0) ? MAXLONG : maxHits;
	requestedNumHits = MIN(maxHits, totalNumHits);

	/* If not enough headlines were returned, keep asking for more headlines. */
	while (currentNumHits < requestedNumHits) {
		/* Send a request for the next set of headlines. */
		as = appConnNextHeadlinesReqWrite(conn,
						  1 + currentNumHits,
				       requestedNumHits - currentNumHits,
						  NULL, NULL);

		if (as == appConnStatusUndefinedFunction)
			break;
		if (*diags != NULL)
			return (false);

		/* Receive the response. */
		while (((as = appConnPduInfo(conn, &pduType, NULL, diags))
			== appConnStatusNotFinished) && (*diags == NULL));

		if (*diags != NULL)
			return (false);

		/* Make sure the response has the correct packet type. */
		if (pduType != appConnNextHeadlinesResp)
			return (false);

		/* Read the response and get the headliness. */
		while (((as = appConnNextHeadlinesRespRead(conn, hits, NULL, diags))
			== appConnStatusNotFinished) && (*diags == NULL));

		if (*diags != NULL)
			return (false);

		/* Update the current headline count. */
		currentNumHits = length(*hits);
	}

	return (true);
}


/*---------------------------------------------------------------------------
  ---------------------------------------------------------------------------
 Function:   displaySearchResults
 Purpose:    Displays search results (hits) to stdout.
 Called by:  Test functions: testSearchAndRetrieveByRecId
                             testSearchWithRFAndRetrieveByPosition
 Parameters: hits -- input; a list of hits where each hit is of datatype
                     appConnHeadline.
 Returns:    void
---------------------------------------------------------------------------*/
static void displaySearchResults(FILE * fp, list hits)
{
	appConnHeadline *hit;
	appConnElement *element;
	appConnVariant *variant;
	long i, j, k;

	for (i = 0; i < length(hits); i++) {
		long count = 0, len = 0;
#ifdef DEBUG
		printf("Displaying Hit %d\n", i);
#endif
		hit = nth(hits, i);

		for (j = 0; j < length(hit->elements); j++) {
			element = nth(hit->elements, j);

			/* If only one variant exists. */
			if (length(element->variants) == 1) {
				variant = first(element->variants);
				len = variant->size;
#ifdef DEBUG
		printf("length is is only one variant: %d\n", len);
#endif
			} else {
#ifdef DEBUG
		printf("length is in many variants:\n");
#endif
				for (k = 0; k < length(element->variants); k++) {
					variant = nth(element->variants, k);
					len += variant->size;
#ifdef DEBUG
		printf("adding variant: %d = %d\n", variant->size, len);
#endif