client.c

harvest是一个下载html网页得机器人

/* 
 * Copyright (c) 1995-2003, Index Data
 * See the file LICENSE for details.
 *
 * $Id: client.c,v 1.208 2003/09/04 18:51:49 adam Exp $
 */

#include <stdio.h>
#include <stdlib.h>
#include <assert.h>
#if HAVE_LOCALE_H
#include <locale.h>
#endif

#if HAVE_LANGINFO_H
#include <langinfo.h>
#endif

#include <time.h>
#include <ctype.h>

#ifdef WIN32
#include <io.h>
#define S_ISREG(x) (x & _S_IFREG)
#define S_ISDIR(x) (x & _S_IFDIR)
#endif

#include <yaz/yaz-util.h>

#include <yaz/comstack.h>

#include <yaz/proto.h>
#include <yaz/marcdisp.h>
#include <yaz/diagbib1.h>
#include <yaz/otherinfo.h>
#include <yaz/charneg.h>

#include <yaz/pquery.h>
#include <yaz/sortspec.h>

#include <yaz/ill.h>
#include <yaz/srw.h>
#include <yaz/yaz-ccl.h>
#include <yaz/cql.h>

#if HAVE_READLINE_READLINE_H
#include <readline/readline.h>
#include <unistd.h>
#endif
#if HAVE_READLINE_HISTORY_H
#include <readline/history.h>
#endif

#include <sys/stat.h>

#include "admin.h"
#include "tabcomplete.h"

#define C_PROMPT "Z> "

static char *codeset = 0;               /* character set for output */
static int hex_dump = 0;
static ODR out, in, print;              /* encoding and decoding streams */
#if HAVE_XML2
static ODR srw_sr_odr_out = 0;
static Z_SRW_PDU *srw_sr = 0;
#endif
static FILE *apdu_file = 0;
static FILE *ber_file = 0;
static COMSTACK conn = 0;               /* our z-association */
static Z_IdAuthentication *auth = 0;    /* our current auth definition */
char *databaseNames[128];
int num_databaseNames = 0;
static Z_External *record_last = 0;
static int setnumber = -1;              /* current result set number */
static int smallSetUpperBound = 0;
static int largeSetLowerBound = 1;
static int mediumSetPresentNumber = 0;
static Z_ElementSetNames *elementSetNames = 0; 
static int setno = 1;                   /* current set offset */
static enum oid_proto protocol = PROTO_Z3950;      /* current app protocol */
static enum oid_value recordsyntax = VAL_USMARC;
static char *schema = 0;
static int sent_close = 0;
static NMEM session_mem = NULL;         /* memory handle for init-response */
static Z_InitResponse *session = 0;     /* session parameters */
static char last_scan_line[512] = "0";
static char last_scan_query[512] = "0";
static char ccl_fields[512] = "default.bib";
/* ### How can I set this path to use wherever YAZ is installed? */
static char cql_fields[512] = "/usr/local/share/yaz/etc/pqf.properties";
static char *esPackageName = 0;
static char *yazProxy = 0;
static int kilobytes = 1024;
static char *negotiationCharset = 0;
static char *outputCharset = 0;
static char *marcCharset = 0;
static char* yazLang = 0;

static char last_cmd[32] = "?";
static FILE *marc_file = 0;
static char *refid = NULL;
static char *last_open_command = NULL;
static int auto_reconnect = 0;

typedef enum {
    QueryType_Prefix,
    QueryType_CCL,
    QueryType_CCL2RPN,
    QueryType_CQL,
    QueryType_CQL2RPN
} QueryType;

static QueryType queryType = QueryType_Prefix;

static CCL_bibset bibset;               /* CCL bibset handle */
static cql_transform_t cqltrans;	/* CQL qualifier-set handle */

#if HAVE_READLINE_COMPLETION_OVER

#else
/* readline doesn't have this var. Define it ourselves. */
int rl_attempted_completion_over = 0;
#endif

/* set this one to 1, to avoid decode of unknown MARCs  */
#define AVOID_MARC_DECODE 1

#define maxOtherInfosSupported 10
struct {
    int oidval;
    char* value;
} extraOtherInfos[maxOtherInfosSupported];
	

void process_cmd_line(char* line);
char ** readline_completer(char *text, int start, int end);
char *command_generator(const char *text, int state);
char** curret_global_list=NULL;
int cmd_register_tab(const char* arg);

static void close_session (void);

ODR getODROutputStream()
{
    return out;
}

const char* query_type_as_string(QueryType q) 
{
    switch (q) { 
    case QueryType_Prefix: return "prefix (RPN sent to server)";
    case QueryType_CCL: return "CCL (CCL sent to server) ";
    case QueryType_CCL2RPN: return "CCL -> RPN (RPN sent to server)";
    case QueryType_CQL: return "CQL (CQL sent to server)";
    case QueryType_CQL2RPN: return "CQL -> RPN (RPN sent to server)";
    default: 
        return "unknown Query type internal yaz-client error";
    }
}

static void do_hex_dump(const char* buf, int len) 
{
    if (hex_dump)
    {
	int i,x;
	for( i=0; i<len ; i=i+16 ) 
	{			
	    printf(" %4.4d ",i);
	    for(x=0 ; i+x<len && x<16; ++x) 
	    {
		printf("%2.2X ",(unsigned int)((unsigned char)buf[i+x]));
	    }
	    printf("\n");
	}
    }
}

void add_otherInfos(Z_APDU *a) 
{
    Z_OtherInformation **oi;
    int i;
		
    yaz_oi_APDU(a, &oi);
    for(i=0; i<maxOtherInfosSupported; ++i) 
    {
        if(extraOtherInfos[i].oidval != -1) 
            yaz_oi_set_string_oidval(oi, out, extraOtherInfos[i].oidval,
                                     1, extraOtherInfos[i].value);
    }   
}

int send_apdu(Z_APDU *a)
{
    char *buf;
    int len;
    
    add_otherInfos(a);
    
    if (apdu_file)
    {
        z_APDU(print, &a, 0, 0);
        odr_reset(print);
    }
    if (!z_APDU(out, &a, 0, 0))
    {
        odr_perror(out, "Encoding APDU");
        close_session();
        return 0;
    }
    buf = odr_getbuf(out, &len, 0);
    if (ber_file)
        odr_dumpBER(ber_file, buf, len);
    /* printf ("sending APDU of size %d\n", len); */
    if (cs_put(conn, buf, len) < 0)
    {
        fprintf(stderr, "cs_put: %s", cs_errmsg(cs_errno(conn)));
        close_session();
        return 0;
    }
    do_hex_dump(buf,len);
    odr_reset(out); /* release the APDU structure  */
    return 1;
}

static void print_stringn(const unsigned char *buf, size_t len)
{
    size_t i;
    for (i = 0; i<len; i++)
        if ((buf[i] <= 126 && buf[i] >= 32) || strchr ("\n\r\t\f", buf[i]))
            printf ("%c", buf[i]);
        else
            printf ("\\X%02X", buf[i]);
}

static void print_refid (Z_ReferenceId *id)
{
    if (id)
    {
        printf ("Reference Id: ");
        print_stringn (id->buf, id->len);
        printf ("\n");
    }
}

static Z_ReferenceId *set_refid (ODR out)
{
    Z_ReferenceId *id;
    if (!refid)
        return 0;
    id = (Z_ReferenceId *) odr_malloc (out, sizeof(*id));
    id->size = id->len = strlen(refid);
    id->buf = (unsigned char *) odr_malloc (out, id->len);
    memcpy (id->buf, refid, id->len);
    return id;
}   

/* INIT SERVICE ------------------------------- */

static void send_initRequest(const char* type_and_host)
{
    Z_APDU *apdu = zget_APDU(out, Z_APDU_initRequest);
    Z_InitRequest *req = apdu->u.initRequest;

    ODR_MASK_SET(req->options, Z_Options_search);
    ODR_MASK_SET(req->options, Z_Options_present);
    ODR_MASK_SET(req->options, Z_Options_namedResultSets);
    ODR_MASK_SET(req->options, Z_Options_triggerResourceCtrl);
    ODR_MASK_SET(req->options, Z_Options_scan);
    ODR_MASK_SET(req->options, Z_Options_sort);
    ODR_MASK_SET(req->options, Z_Options_extendedServices);
    ODR_MASK_SET(req->options, Z_Options_delSet);

    ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_1);
    ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_2);
    ODR_MASK_SET(req->protocolVersion, Z_ProtocolVersion_3);

    *req->maximumRecordSize = 1024*kilobytes;
    *req->preferredMessageSize = 1024*kilobytes;

    req->idAuthentication = auth;

    req->referenceId = set_refid (out);

    if (yazProxy) 
        yaz_oi_set_string_oidval(&req->otherInfo, out, VAL_PROXY,
        1, type_and_host);
    
    if (negotiationCharset || yazLang) {
    	Z_OtherInformation **p;
    	Z_OtherInformationUnit *p0;
    	
    	yaz_oi_APDU(apdu, &p);
    	
    	if ((p0=yaz_oi_update(p, out, NULL, 0, 0))) {
    		ODR_MASK_SET(req->options, Z_Options_negotiationModel);
    		
    		p0->which = Z_OtherInfo_externallyDefinedInfo;
    		p0->information.externallyDefinedInfo =
    			yaz_set_proposal_charneg(
                            out,
                            (const char**)&negotiationCharset, 
                            negotiationCharset ? 1 : 0,
                            (const char**)&yazLang, yazLang ? 1 : 0, 1);
    	}
    }
    
    if (send_apdu(apdu))
        printf("Sent initrequest.\n");
}

static int process_initResponse(Z_InitResponse *res)
{
    int ver = 0;
    /* save session parameters for later use */
    session_mem = odr_extract_mem(in);
    session = res;

    for (ver = 0; ver<5; ver++)
        if (!ODR_MASK_GET(res->protocolVersion, ver))
            break;

    if (!*res->result)
        printf("Connection rejected by v%d target.\n", ver);
    else
        printf("Connection accepted by v%d target.\n", ver);
    if (res->implementationId)
        printf("ID     : %s\n", res->implementationId);
    if (res->implementationName)
        printf("Name   : %s\n", res->implementationName);
    if (res->implementationVersion)
        printf("Version: %s\n", res->implementationVersion);
    if (res->userInformationField)
    {
	Z_External *uif = res->userInformationField;
        printf("UserInformationfield:\n");
        if (!z_External(print, (Z_External**)&uif, 0, 0))
        {
            odr_perror(print, "Printing userinfo\n");
            odr_reset(print);
        }
        if (uif->which == Z_External_octet)
        {
            printf("Guessing visiblestring:\n");
            printf("'%s'\n", uif->u. octet_aligned->buf);
        } else if (uif->which == Z_External_single) {
	    /* Peek at any private Init-diagnostic APDUs */
	    Odr_any *sat = uif->u.single_ASN1_type;
	    printf("### NAUGHTY: External is '%s'\n", sat->buf);
	}
        odr_reset (print);
    }
    printf ("Options:");
    if (ODR_MASK_GET(res->options, Z_Options_search))
        printf (" search");
    if (ODR_MASK_GET(res->options, Z_Options_present))
        printf (" present");
    if (ODR_MASK_GET(res->options, Z_Options_delSet))
        printf (" delSet");
    if (ODR_MASK_GET(res->options, Z_Options_resourceReport))
        printf (" resourceReport");
    if (ODR_MASK_GET(res->options, Z_Options_resourceCtrl))
        printf (" resourceCtrl");
    if (ODR_MASK_GET(res->options, Z_Options_accessCtrl))
        printf (" accessCtrl");
    if (ODR_MASK_GET(res->options, Z_Options_scan))
        printf (" scan");
    if (ODR_MASK_GET(res->options, Z_Options_sort))
        printf (" sort");
    if (ODR_MASK_GET(res->options, Z_Options_extendedServices))
        printf (" extendedServices");
    if (ODR_MASK_GET(res->options, Z_Options_level_1Segmentation))
        printf (" level1Segmentation");
    if (ODR_MASK_GET(res->options, Z_Options_level_2Segmentation))
        printf (" level2Segmentation");
    if (ODR_MASK_GET(res->options, Z_Options_concurrentOperations))
        printf (" concurrentOperations");
    if (ODR_MASK_GET(res->options, Z_Options_namedResultSets))
    {
        printf (" namedResultSets");
        setnumber = 0;
    }
    if (ODR_MASK_GET(res->options, Z_Options_encapsulation))
        printf (" encapsulation");
    if (ODR_MASK_GET(res->options, Z_Options_resultCount))
        printf (" resultCount");
    if (ODR_MASK_GET(res->options, Z_Options_negotiationModel))
        printf (" negotiationModel");
    if (ODR_MASK_GET(res->options, Z_Options_duplicateDetection))
        printf (" duplicateDetection");
    if (ODR_MASK_GET(res->options, Z_Options_queryType104))
        printf (" queryType104");
    printf ("\n");
    
    if (ODR_MASK_GET(res->options, Z_Options_negotiationModel)) {
    
    	Z_CharSetandLanguageNegotiation *p =
    		yaz_get_charneg_record(res->otherInfo);
    	
    	if (p) {
            
            char *charset=NULL, *lang=NULL;
            int selected;
            
            yaz_get_response_charneg(session_mem, p, &charset, &lang,
                                     &selected);

	    if (outputCharset && negotiationCharset) {
		    odr_set_charset (out, charset, outputCharset);
		    odr_set_charset (in, outputCharset, charset);
	    }
	    else {
		    odr_set_charset (out, 0, 0);
		    odr_set_charset (in, 0, 0);
	    }
            
            printf("Accepted character set : %s\n", charset);
            printf("Accepted code language : %s\n", lang ? lang : "none");
            printf("Accepted records in ...: %d\n", selected );
    	}
    }
    fflush (stdout);
    return 0;
}

static int set_base(const char *arg)
{
    int i;
    const char *cp;

    for (i = 0; i<num_databaseNames; i++)
        xfree (databaseNames[i]);
    num_databaseNames = 0;
    while (1)
    {
        char *cp1;
        if (!(cp = strchr(arg, ' ')))
            cp = arg + strlen(arg);
        if (cp - arg < 1)
            break;
        databaseNames[num_databaseNames] = (char *)xmalloc (1 + cp - arg);
        memcpy (databaseNames[num_databaseNames], arg, cp - arg);
        databaseNames[num_databaseNames][cp - arg] = '\0';

        for (cp1 = databaseNames[num_databaseNames]; *cp1 ; cp1++)
            if (*cp1 == '+')