index.c

harvest是一个下载html网页得机器人

    FREEP(ptr1);
    FREEP(NEB_context);
    NEB_context = ptr2;

    return (SUCCESS);
}

/* ----------------------------------------------------------------- *
 * NEB_bulk_query -- process a bulk query, return full SOIF objects.
 * ----------------------------------------------------------------- */
LOCAL int NEB_bulk_query(rsock, ptime, head)
     int rsock;
     time_t ptime;
     POINTER head;
{
    FILE *fp;
    POINTER ptr;
    fd_t fd;
    reg_t *entry;

    if ((fp = fdopen(rsock, "w")) == NULL) {
	perror("fdopen");
	QM_send_bulk_err(rsock);
	return ERROR;
    }
    QM_send_bulk_begin(rsock);

    for (ptr = head; ptr != NIL; ptr = CDR(ptr)) {
	fd = GETFIXNUM(CAR(ptr));
	if ((entry = RG_Get_Entry(fd)) != NULL)
	    if (entry->update_time >= ptime)
		QM_send_bulk_fd(fd, fp, entry);
    }
    fflush(fp);
    QM_send_bulk_end(rsock);

    fclose(fp);
    return SUCCESS;
}

/* ----------------------------------------------------------------- *
 * NEB_del_query -- delete all objects that match the query.
 * ----------------------------------------------------------------- */
LOCAL int NEB_del_query(rsock, head)
     int rsock;
     POINTER head;
{
    POINTER ptr;
    fd_t fd;
    reg_t *entry;

    for (ptr = head; ptr != NIL; ptr = CDR(ptr)) {
	fd = GETFIXNUM(CAR(ptr));
	if ((entry = RG_Get_Entry(fd) != NULL)) {
	    COL_DEL_Obj(entry);
	}
	return SUCCESS;
    }
}

/* ----------------------------------------------------------------- *
 * NEB_user_query -- Read the output of the Nebula query from the pointer
 * head then send to rsock via protocol.
 * ----------------------------------------------------------------- */
LOCAL int NEB_user_query(rsock, head)
     int rsock;
     POINTER head;
{
    POINTER ptr;
    fd_t fd;
    int obcnt = 0;

    for (ptr = head; ptr != NIL; ptr = CDR(ptr)) {
	fd = GETFIXNUM(CAR(ptr));
	QM_user_object(rsock, fd, 0, NULL);
	obcnt++;
    }

    QM_user_done(rsock, obcnt);

    return SUCCESS;
}

/* ----------------------------------------------------------------- *
 * NEB_qprocess -- Recursive function to build a query from the list.
 * ----------------------------------------------------------------- */
LOCAL POINTER NEB_qbuild(ql)
     qlist_t *ql;
{
    POINTER ptr;

    if (ql == NULL)
	return (NIL);

    switch (ql->type) {
    case LOGICAL:
	switch (ql->op) {
	case AND:
	    ptr = CVLIST(NEB_qbuild(ql->rlist), NIL);
	    ptr = CVLIST(NEB_qbuild(ql->llist), ptr);
	    ptr = CVLIST(CVSYM("INTERSECT-SET"), ptr);
	    return (ptr);

	case OR:
	    ptr = CVLIST(NEB_qbuild(ql->rlist), NIL);
	    ptr = CVLIST(NEB_qbuild(ql->llist), ptr);
	    ptr = CVLIST(CVSYM("UNION-SET"), ptr);
	    return (ptr);

	case NOT:
	    ptr = CVLIST(NEB_qbuild(ql->llist), NIL);
	    ptr = CVLIST(CVSYM("NOT"), ptr);
	    return (ptr);

	case EXCEPT:
	    ptr = CVLIST(NEB_qbuild(ql->rlist), NIL);
	    ptr = CVLIST(NEB_qbuild(ql->llist), ptr);
	    ptr = CVLIST(CVSYM("DIFF-SET"), ptr);
	    return (ptr);

	default:
#if DEBUG0
	    fprintf(stderr, "Unknown logical operation %d.\n", (int) ql->op);
#endif
	    return (NIL);
	}

    case SELECT:
	switch (ql->op) {
	case EXACT:
	    ptr = CVLIST(CVSTRING(ql->rlist), NIL);
	    if (ql->llist != NULL)
		ptr = CVLIST(CVSTRING(ql->llist), ptr);
	    else
		ptr = CVLIST(CVSTRING("*keys*"), ptr);
	    ptr = CVLIST(CVSYM("HARVEST-EQUAL"), ptr);
	    return (ptr);

	case REGEX:
	    ptr = CVLIST(CVSTRING(ql->rlist), NIL);
	    if (ql->llist != NULL)
		ptr = CVLIST(CVSTRING(ql->llist), ptr);
	    else
		ptr = CVLIST(CVSTRING("*keys*"), ptr);
	    ptr = CVLIST(CVSYM("HARVEST-REGEX"), ptr);
	    return (ptr);

	case GTT:
	    ptr = CVLIST(CVSTRING(ql->rlist), NIL);
	    if (ql->llist != NULL)
		ptr = CVLIST(CVSTRING(ql->llist), ptr);
	    else
		ptr = CVLIST(CVSTRING("*keys*"), ptr);
	    ptr = CVLIST(CVSYM("HARVEST-GT"), ptr);
	    return (ptr);

	case LST:
	    ptr = CVLIST(CVSTRING(ql->rlist), NIL);
	    if (ql->llist != NULL)
		ptr = CVLIST(CVSTRING(ql->llist), ptr);
	    else
		ptr = CVLIST(CVSTRING("*keys*"), ptr);
	    ptr = CVLIST(CVSYM("HARVEST-LT"), ptr);
	    return (ptr);

	default:
#if DEBUG0
	    fprintf(stderr, "Unknown select operation %d.\n", (int) ql->op);
#endif
	    return (NIL);
	}

    default:
#if DEBUG0
	fprintf(stderr, "Unknown operation type %d.\n", (int) ql->type);
#endif
	return (NIL);
    }
}

/* ----------------------------------------------------------------- *
 * NEB_qprocess -- Process the query.
 * ----------------------------------------------------------------- */
POINTER NEB_qprocess(query)
     POINTER query;
{
    static int fclass = -1;
    static int fhandle = -1;
    POINTER ptr1, ptr2;

    /* ----- Initialization ----- */
    unpi_error = NULL;
    if (uptr == NULL)
	NEB_initcon();

    if (fclass == -1) {
	if (!unpi_getfunction(uptr, "harvest::process-query", &fclass, &fhandle))
	    return NIL;
#if DEBUG2
	fprintf(stdout, "harvest::process-query: %d %d\n", fclass, fhandle);
#endif
    }
    /* ----- Create argument list ----- */
    ptr1 = CVLIST(query, NIL);

    /* ----- Process RPC ----- */
    ptr2 = ccall(uptr->channel, fclass, fhandle, NEB_key, NEB_context, ptr1, NIL);
    if ((ptr2 == NIL) && (unpi_error != NULL)) {
	fprintf(stderr, "Nebula process-query failed: %s\n", (char *) unpi_error);
	FREEP(ptr1);
	free(unpi_error);
	return NIL;
    }
#if DEBUG2
    fprintf(stdout, "Successfully processed query:\n");
    writeexp(stdout, ptr2);
    unpputc(stdout, '\n');
#endif

    /* ----- Clean up ----- */
    FREEP(ptr1);

    return (ptr2);
}


/* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX *
 * 
 * PUBLIC FUNCTIONS
 * 
 * XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX */

/* ----------------------------------------------------------------- *
 * info -- UNP info function
 * ----------------------------------------------------------------- */
info(fmt, msg)
     char *fmt, *msg;
{
    extern char *unpi_error;
    static char buf[STRMAX];

    sprintf(buf, fmt, msg);
#if DEBUG2
    fprintf(stderr, "%s\n", buf);
#endif
    unpi_error = (char *) strdup(buf);
}


/* ----------------------------------------------------------------- *
 * IND_New_Object -- index a new object
 * ----------------------------------------------------------------- */
int Nebula_IND_New_Object(entry)
     reg_t *entry;
{
    NEBq *qn;

    switch (IndexType) {
    case I_FULL:
	/* Ignore, we'll get it later. */
	return (SUCCESS);

    case I_INCR:
	qn = (NEBq *) xmalloc(sizeof(NEBq));
	qn->fname = SM_Get_Obj_Filename(entry->FD);
	qn->next = NEB_indexq;
	NEB_indexq = qn;
	return (SUCCESS);

    case I_PER_OBJ:
	return (NEB_Index_Object(entry));

    default:
	/* SHOULDN'T HAPPEN!! */
	return (ERROR);
    }
}

/* ----------------------------------------------------------------- *
 * INDIndex_Full -- builds an index of all of the object files.
 * ----------------------------------------------------------------- */
int Nebula_IND_Index_Full()
{
    return (NEB_Index_Full());
}
/* ----------------------------------------------------------------- *
 * IND_Index_Incremental -- do a full incremental update of the index.
 * ----------------------------------------------------------------- */
int Nebula_IND_Index_Incremental()
{
    return (NEB_Index_Batch());
}

/* ----------------------------------------------------------------- *
 * IND_Index_Start -- prepare for indexing a stream of objects.
 * ----------------------------------------------------------------- */
int Nebula_IND_Index_Start()
{
    NEB_indexq = NEB_deleteq = NULL;
}

/* ----------------------------------------------------------------- *
 * IND_Index_Flush -- finish indexing a stream of objects.
 * ----------------------------------------------------------------- */
int Nebula_IND_Index_Flush()
{
    int stat = SUCCESS;

    switch (IndexType) {
    case I_FULL:
	return (NEB_Index_Full());

    case I_INCR:
	if (NEB_deleteq != NULL)
	    if (NEB_Destroy_Batch() == ERROR)
		stat = ERROR;
	if (NEB_indexq != NULL)
	    if (NEB_Index_Batch() == ERROR)
		stat = ERROR;
	return (stat);

    case I_PER_OBJ:
	return (SUCCESS);

    default:
	/* SHOULDN'T HAPPEN!! */
	return (ERROR);
    }
}

/* ----------------------------------------------------------------- *
 * IND_Destroy_Obj -- remove an object from the indexer.
 * ----------------------------------------------------------------- */
int Nebula_IND_Destroy_Obj(entry)
     reg_t *entry;
{
    NEBq *qn;

    switch (IndexType) {
    case I_FULL:
	/* Ignore, we'll get it later. */
	return (SUCCESS);

    case I_INCR:
	qn = (NEBq *) xmalloc(sizeof(NEBq));
	qn->fname = SM_Get_Obj_Filename(entry->FD);
	qn->next = NEB_deleteq;
	NEB_deleteq = qn;
	return (SUCCESS);

    case I_PER_OBJ:
	return (NEB_Destroy_Object(entry));

    default:
	/* SHOULDN'T HAPPEN!! */
	return (ERROR);
    }
}

/* ----------------------------------------------------------------- *
 * IND_initialize -- initialize interface to indexer
 * ----------------------------------------------------------------- */
int Nebula_IND_initialize()
{
    extern char *HName;
    int context;

    if ((NEB_cname = strdup("broker")) == NULL)
	fatal("strdup failed");

    if ((NEB_host = strdup(HName)) == NULL)
	fatal("strdup failed");

    return (SUCCESS);
}

/* ----------------------------------------------------------------- *
 * IND_Init_Flags -- initialize parser flags
 * ----------------------------------------------------------------- */
void Nebula_IND_Init_Flags()
{
}

/* ----------------------------------------------------------------- *
 * IND_Set_Flags -- set query parser flag
 * ----------------------------------------------------------------- */
void Nebula_IND_Set_Flags(flag, val)
     char *flag;
     char *val;
{
}

/* ----------------------------------------------------------------- *
 * IND_config -- configure indexer specific variables
 * ----------------------------------------------------------------- */
int Nebula_IND_config(value, tag)
     char *value;
     char *tag;
{
#if DEBUG1
    if (tag != NULL)
	printf("Nebula Configuration: %s %s\n", value, tag);
#endif

    if (tag != NULL) {
	if (strcasecmp(tag, NEB_HOST) == 0) {
	    if ((NEB_host = strdup(value)) == NULL)
		fatal("strdup failed");
	    return SUCCESS;
	} else if (strcasecmp(tag, NEB_PORT) == 0) {
	    if (sscanf(value, "%d", &NEB_port) < 1) {
		fprintf(stderr, "Could not convert Nebula port. \n");
		return ERROR;
	    }
	    if (NEB_port < 0) {
		fprintf(stderr, "Invalid Nebula port number %d.\n", NEB_port);
		NEB_port = 0;
		return ERROR;
	    }
	    return SUCCESS;
	} else if (strcasecmp(tag, NEB_KEY) == 0) {
	    if (sscanf(value, "%d", &NEB_key) < 1) {
		fprintf(stderr, "Count not convert Nebula key.\n");
		return ERROR;
	    }
	    return SUCCESS;
	} else if (strcasecmp(tag, NEB_CNAME) == 0) {
	    if ((NEB_cname = strdup(value)) == NULL)
		fatal("strdup failed");
	    return SUCCESS;
	}
    }				/* tag != NULL */
    return ERROR;
}

/* ----------------------------------------------------------------- *
 * IND_do_query -- construct a query based upon the query list 
 * structure.
 * ----------------------------------------------------------------- */
int Nebula_IND_do_query(ql, rsock, qflag, ptime)
     qlist_t *ql;
     int rsock;
     int qflag;
     time_t ptime;
{
    POINTER ptr1, ptr2;

    if ((ptr1 = NEB_qbuild(ql)) != NIL)
	ptr2 = NEB_qprocess(ptr1);
    else
	ptr2 = NIL;

    switch (qflag) {
    case UQUERY:
	return (NEB_user_query(rsock, ptr2));

    case QBULK:
	return (NEB_bulk_query(rsock, ptime, ptr2));

    case QDELETE:
	return (NEB_del_query(rsock, ptr2));

    default:
#if DEBUG0
	fprintf(stderr, "Invalid query type %d.\n", qflag);
#endif
	return ERROR;
    }
}