waisquery.c

harvest是一个下载html网页得机器人

				}
			}
		}
#ifdef DEBUG
		printf("RESULT: %3ld   score %4ld len %ld %s\n",
			hit->rank, hit->score, len < 0 ? -len : len,
			hit->headline);
#endif
		fprintf(fp, "%3ld   score %4ld len %ld %s\n",
			hit->rank, hit->score, len < 0 ? -len : len,
			hit->headline);
	}
	fflush(fp);
}



/*---------------------------------------------------------------------------
  ---------------------------------------------------------------------------
 Function:   displayDiagsAndCleanup
 Purpose:    Display the diagnostics to stdout, and frees qry, hits, data, 
             and diags.
 Called by:  Test functions: testSearchAndRetrieveByRecId
                             testSearchWithRFAndRetrieveByPosition
 Parameters: qry   -- input; a handle to an appConnQry data type created
                      by the appQryAddTerm() function.
             hits  -- input; a handle to a list.  Each item in the list is
                      of type appConnHeadline.
             data  -- input; a character string containing the retrieved data
                      to be freed.
             diags -- output; a handle to a list of diagnostics, where each
                      item in the list is of type appConnDiag (see 
                      appConnMgr.c).  If the operation was unsuccessful, 
                      a list of diags are returned in this parameter.
 Returns:    void
---------------------------------------------------------------------------*/
static void displayDiagsAndCleanup(appConnQry * qry,
				   list hits,
				   char *data,
				   list diags)
{
	appConnDiag *diag;
	long i;

	for (i = 0L, diag = nth(diags, 0L); i < length(diags); i++, diag = nth(diags, i))
		printf("Diagnostic code %ld, %s\n", diag->code, diag->info);

#ifdef DEBUG
	printf("Cleaning up the query...\n");
#endif

	/* Clean up. */
	appConnQryFree(qry);

	/* Free the hits from the search. */
	for (i = 0L; i < length(hits); i++) {
		appConnHeadlineFree(nth(hits, i));
#ifdef DEBUG
		printf("Cleaning up query hit %d...\n", i);
#endif
	}
	freeList(hits);

	safeFree(data);

	/* Free the diags. */
	for (i = 0L; i < length(diags); i++) {
		appConnDiagFree(nth(diags, i));
#ifdef DEBUG
		printf("Cleaning up diag %d...\n", i);
#endif
	}
	freeList(diags);

}

/*----------------------------------------------------------------------*/
/*---------------------- Harvest Functions -----------------------------*/
/*----------------------------------------------------------------------*/

/* 
 *  teardown_search() - Kills the connection to the WAIS server
 */
void teardown_search()
{
#ifdef DEBUG
	printf("teardown_search: cleaning up state...\n");
#endif
	/* clean up state */
	if (theDBname)
		free(theDBname);
	theDBname = NULL;
	if (theHost)
		free(theHost);
	theHost = NULL;
	thePort = -1;

	if (conn == NULL)
		return;		/* nothing to do */

#ifdef DEBUG
	printf("teardown_search: cleaning up connection...\n");
#endif
	/* Clean up TCP nConn connection and Application appConn connection. */
	nConnFree(net);
	appConnFree(conn);
	nConnTypesFree();
	appConnTypesFree();

	conn = NULL;		/* reset state */
}

/*
 *  perform_search - send query to server, and print results
 */
static void perform_search(FILE * fp, char *qryTxt, int maxhits)
{
	appConnQry *qry = NULL;
	list hits = NULL;
	list diags = NULL;
	long i;

	/* Create query, search, and display results. */
	qry = appConnQryMakeLeafFreeText(qryTxt);
	if (search(conn, (long) maxhits, qry, NULL, &hits, &diags)) {
#ifdef DEBUG
		printf("Got %d hits (max %d)\n", length(hits), maxhits);
#endif
		displaySearchResults(fp, hits);
	} else {
		printf("Search (%s) failed.\n", qryTxt);
	}
	displayDiagsAndCleanup(qry, hits, NULL, diags);
}

/*
 *  setup_search - establish a connection to the wais server.
 */
static int setup_search(char *host, long port)
{
	char *nConnTypeName = "TCP";
	char *appConnTypeName = "Z39501988";	/* .v2 breaks! */
	list diags = NULL;

	/* force a reset if needed */
	if (conn != NULL)
		teardown_search();

	/* set state */
	thePort = port;
	if (theHost)
		free(theHost);
	theHost = strdup(host);

#ifdef DEBUG
	printf("Register the client application-layer protocols.\n");
#endif
	/* Register the client application-layer protocols. */
	appConnClientInit();

#ifdef DEBUG
	printf("Open a TCP connection.\n");
#endif
	/* Open a TCP connection. */
	if (nConnOpen(&net, nConnTypeName, 2L, theHost, thePort) !=
	    nConnStatusOK) {
		printf("Unable to open a %s network connection to %s:%ld.\n",
		       nConnTypeName, theHost, thePort);
		nConnTypesFree();
		appConnTypesFree();
		return 1;
	}
#ifdef DEBUG
	printf("Using the TCP conn, open a application protocol conn.\n");
#endif
	/* Using the TCP connection, open a application protocol connection. */
	if (appConnOpen(&conn, appConnTypeName, net, theBuffer,
			BUFFER_SIZE, 1L, 0L) != appConnStatusOK) {
		printf("Unable to open a %s application connection.\n",
		       appConnTypeName);
		nConnTypesFree();
		appConnTypesFree();
		nConnFree(net);
		return 1;
	}
#ifdef DEBUG
	printf("Initialize and display any diagnostics that result.\n");
#endif
	/* Initialize and display any diagnostics that result. */
	if (!init(conn, CLIENT_DESC, &diags)) {
		printf("Cannot initialize %s connection using protocol %s.\n",
		       nConnTypeName, appConnTypeName);
		displayDiagsAndCleanup(NULL, NULL, NULL, diags);
		nConnTypesFree();
		appConnTypesFree();
		nConnFree(net);
		appConnFree(conn);
		return 1;
	}
#ifdef DEBUG
	printf("setup_search done\n");
#endif
	return 0;
}

/*
 *  do_search - Submits 'query' to the WAIS server running on host:port, 
 *              and writes the result set in fp.
 *
 *  Returns non-zero on error; otherwise, returns 0.
 */
int do_wais_inline_search(FILE * fp,
			  char *host,
			  long port,
			  char *query,
			  char *dbname,
			  int maxhits)
{
#ifdef DEBUG
	printf("CALLING do_wais_inline_search: %p, %s, %d, %s, %s, %d\n",
		fp, host, port, query, dbname, maxhits);
#endif
	/* 
	 *  Init the search, if this is our first time through, or
	 *  the query is on a different database that the last one 
	 */
	if ((conn == NULL) ||
	    (strcmp(host, theHost) != 0) ||
	    (port != thePort) ||
	    (strcmp(dbname, theDBname) != 0)) {
		printf("Switching to WAIS server (%s:%d) for %s...\n",
		       host, port, dbname);
		if (setup_search(host, port)) {
			printf("ERROR: Cannot connect to WAIS server %s:%d.\n",
			       host, port);
			teardown_search();	/* force reset */
			return 1;
		}
		theDBname = strdup(dbname);
	}
#ifdef DEBUG
	else
		printf("Doing inline search using previous connection: %s\n", query);
#endif
	/* do the search, and send results to fp */
	perform_search(fp, query, maxhits);

#ifdef USE_CONN_PER_QUERY
	/* this will reset the connection for each query */
	teardown_search();
#endif

	return 0;
}

#ifdef USE_MAIN
/* simple test program.  submit queries to stdin. */
int main(argc, argv)
int argc;
char *argv[];
{
	char buf[BUFSIZ];
	if (argc != 5) {
		printf("Usage: %s host port dbname maxhits\nqueries to stdin",
		       argv[0]);
		exit(1);
	}
	while (fgets(buf, BUFSIZ, stdin)) {
		do_wais_inline_search(stdout,
				      argv[1],
				      atoi(argv[2]),
				      buf,
				      argv[3],
				      atoi(argv[4]));
	}
	teardown_search();
	exit(0);
}
#endif