split.c

harvest是一个下载html网页得机器人

			n->op = 0;
			n->type = LEAF;
			n->terminalindex = (*pnum_terminals);
			n->data.leaf.value = NULL;
			n->data.leaf.attribute = 0;
			if (!get_attribute_value((int *)&n->data.leaf.attribute, &n->data.leaf.value, tokenbuf, tokenlen, num_attr)) return NULL;
			strcpy(&apattern[*apatptr], n->data.leaf.value);
			*apatptr += strlen(n->data.leaf.value);
			(*pnum_terminals)++;
			n->op |= NOTPAT;
			t = n;
			break;

		case '{':
			apattern[(*apatptr)++] = token;
			if ((t = parse_tree(buffer, len, bufptr, apattern, apatptr, terminals, pnum_terminals, num_attr)) == NULL) return NULL;
			if (t->op & NOTPAT) t->op &= ~NOTPAT;
			else t->op |= NOTPAT;
			if ((token = get_token_bool(buffer, len, bufptr, tokenbuf, &tokenlen)) != '}') {
				fprintf(stderr, "%s: parse error at offset %d\n", GProgname, *bufptr);
				destroy_tree(t);
				return NULL;
			}
			apattern[(*apatptr)++] = '}';
			break;

		default:
			fprintf(stderr, "%s: parse error at offset %d\n", GProgname, *bufptr);
			return NULL;
		}
		/* The resulting tree is in t. Now do another lookahead at this level */

		if ((token = get_token_bool(buffer, len, bufptr, tokenbuf, &tokenlen)) == 'e') return t;
		switch(token)
		{
		/* must find boolean infix operator */
		case ',':
		case ';':
			apattern[(*apatptr)++] = token;
			leftn = t;
			if ((t = parse_tree(buffer, len, bufptr, apattern, apatptr, terminals, pnum_terminals, num_attr)) == NULL) return NULL;
			n = (ParseTree *)malloc(sizeof(ParseTree));
			n->op = (token == ';') ? ANDPAT : ORPAT ;
			n->type = INTERNAL;
			n->data.internal.left = leftn;
			n->data.internal.right = t;
			return n;

		case '}':
			unget_token_bool(bufptr, tokenlen);
			return t;

		default:
			destroy_tree(t);
			fprintf(stderr, "%s: parse error at offset %d\n", GProgname, *bufptr);
			return NULL;
		}

	case	'a':	/* individual term (attr=val) */
		if (tokenlen == 0) return NULL;
		memcpy(oldtokenbuf, tokenbuf, tokenlen);
		oldtokenlen = tokenlen;
		oldtokenbuf[oldtokenlen] = '\0';
		token = get_token_bool(buffer, len, bufptr, tokenbuf, &tokenlen);
		switch(token)
		{
		case '}':	/* part of case '{' above: else syntax error not detected but semantics ok */
			unget_token_bool(bufptr, tokenlen);
		case 'e':	/* endof input */
		case ',':
		case ';':
			if (*pnum_terminals >= MAXNUM_PAT) {
				fprintf(stderr, "%s: pattern expression too long (> %d terms)\n", GProgname, MAXNUM_PAT);
				return NULL;
			}
			n = &terminals[*pnum_terminals];
			n->op = 0;
			n->type = LEAF;
			n->terminalindex = (*pnum_terminals);
			n->data.leaf.value = NULL;
			n->data.leaf.attribute = 0;
			if (!get_attribute_value((int *)&n->data.leaf.attribute, &n->data.leaf.value, oldtokenbuf, oldtokenlen, num_attr)) return NULL;
			strcpy(&apattern[*apatptr], n->data.leaf.value);
			*apatptr += strlen(n->data.leaf.value);
			(*pnum_terminals)++;
			if ((token == 'e') || (token == '}')) return n;	/* nothing after terminal in expression */

			leftn = n;
			apattern[(*apatptr)++] = token;
			if ((t = parse_tree(buffer, len, bufptr, apattern, apatptr, terminals, pnum_terminals, num_attr)) == NULL) return NULL;
			n = (ParseTree *)malloc(sizeof(ParseTree));
			n->op = (token == ';') ? ANDPAT : ORPAT ;
			n->type = INTERNAL;
			n->data.internal.left = leftn;
			n->data.internal.right = t;
			return n;

		default:
			fprintf(stderr, "%s: parse error at offset %d\n", GProgname, *bufptr);
			return NULL;
		}

	case	'e':	/* can't happen as I always do a lookahead above and return current tree if e */
	default:
		fprintf(stderr, "%s: parse error at offset %d\n", GProgname, *bufptr);
		return NULL;
	}
}

int
split_pattern(GPattern, GM, APattern, terminals, pnum_terminals, pGParse, num_attr)
	CHAR	*GPattern;
	int	GM;
	CHAR	*APattern;
	ParseTree terminals[];
	int	*pnum_terminals;
	ParseTree **pGParse;
	int	num_attr;
{
	int	bufptr = 0, apatptr = 0, ret, i, j;

	foundattr = 0;
	if (is_complex_boolean(GPattern, GM)) {
		ComplexBoolean = 1;
		*pnum_terminals = 0;
		if ((*pGParse = parse_tree(GPattern, GM, &bufptr, APattern, &apatptr, terminals, pnum_terminals, num_attr)) == NULL)
			return -1;
		/* print_tree(*pGParse, 0); */
		APattern[apatptr] = '\0';
		if (foundattr || WHOLEFILESCOPE) {	/* Search in agrep must always be OR since scope is whole file */
			int	i, j;

			for (i=0; i<apatptr; i++) {
				if (APattern[i] == '\\') i++;
				else if (APattern[i] == ';') APattern[i] = ',';
				else if ((APattern[i] == '~') || (APattern[i] == '{') || (APattern[i] == '}')) {
					/* eliminate it from pattern by shifting (including '\0') since agrep must essentially do a flat search */
					if (APattern[i] == '~') foundnot = 1;
					for (j=i; j<apatptr; j++)
						APattern[j] = APattern[j+1];
					apatptr --;
					i--;	/* to counter the ++ on top */
				}
			}
		}
		return *pnum_terminals;
	}
	else {
		for (i=0; i<GM; i++) {
			if (GPattern[i] == '\\') i++;
			else if ((GPattern[i] == '{') || (GPattern[i] == '}')) {
				/* eliminate it from pattern by shifting (including '\0') since agrep must essentially do a flat search */
				for (j=i; j<GM; j++)
					GPattern[j] = GPattern[j+1];
				GM --;
				i--;	/* counter the ++ on top */
			}
		}

		ComplexBoolean = 0;
		*pnum_terminals = 0;
		if ((ret = split_pattern_flat(GPattern, GM, APattern, terminals, pnum_terminals, (int *)pGParse, num_attr)) == -1)
			return -1;
		return ret;
	}
}

int eval_tree();	/* use the one in agrep/asplit.c */

/* MUST CHANGE ALL memgreps TO USE EXEC DIRECTLY IF POSSIBLE (LAST PRIORITY) */

/* All borrowed from main.c and are needed for searching the index */
extern	CHAR	*pat_list[MAXNUM_PAT];  /* complete words within global pattern */
extern	int	pat_lens[MAXNUM_PAT];   /* their lengths */
extern	int	pat_attr[MAXNUM_PAT];   /* set of attributes */
extern	int	num_pat;
extern	CHAR	pat_buf[(MAXNUM_PAT + 2)*MAXPAT];
extern	int	pat_ptr;
extern	int	is_mgrep_pat[MAXNUM_PAT];
extern	int	mgrep_pat_index[MAXNUM_PAT];
extern	int	num_mgrep_pat;
extern	struct offsets **src_offset_table;
extern	struct offsets **curr_offset_table;
extern	char	tempfile[];
extern	int	patindex;
extern	int	patbufpos;
extern	ParseTree terminals[MAXNUM_PAT];
extern	int	num_terminals;
extern int INVERSE;	/* agrep's global: need here to implement ~ in index-search */

/* [first, last) = C-style range for which we want the words in terminal-values' patterns: 0..num_terminals for !ComplexBoolean, term/term otherwise */
split_terminal(first, last)
	int	first, last;
{
        CHAR	*buffer;
        CHAR	*buffer_new;
        CHAR	*buffer_end;
        CHAR	*bp;
        CHAR	word[MAXNAME];
        int	word_length;
        int	type;
	int	i, j, k, attribute;
	char	*substring;	/* used with strstr(superstring, substring) */

	pat_ptr = 0;
	num_mgrep_pat = 0;
	num_pat = 0;

	for (; first<last; first++) {
		attribute = (int)terminals[first].data.leaf.attribute;
		buffer = terminals[first].data.leaf.value;
		buffer_end = buffer + strlen(terminals[first].data.leaf.value);

                /* Now find out which; are the "words" we can search for in the index: each attr_val can have many words in it: e.g., "AUTHOR=Udi Manber" */
                while ((buffer_new = getword("stdin", word, buffer, buffer_end, NULL, NULL)) <= buffer_end) {
                        word_length = strlen(word);
                        if (word_length <= 0) {
                                buffer = buffer_new;
                                if (buffer_new >= buffer_end) break;
                                continue;
                        }
                        if ((type = checksg(word, D, 0)) == -1) return -1;
			if (!type && ComplexBoolean) {
				fprintf(stderr, "%s: query has complex patterns (like '.*') or options (like -n)\n... cannot search for arbitrary booleans\n", GProgname);
				return -1;
			}

#if	0
DISABLED IN GLIMPSE NOW SINCE MGREP HANDLES DUPLICATES -- IT WAS ALWAYS ABLE TO HANDLE SUPERSTRINGS/SUBSTRINGS...: bgopal, Nov 19, 1996
			if (type) {
				/* Check if superstring: if so, ditch word */
				for (i=0; i<num_pat; i++) {
					if (!is_mgrep_pat[i]) continue;
					substring = strstr(word, pat_list[i]);
					if ((substring != NULL) && (substring[0] != '\0')) break;
				}
				if (i < num_pat) {	/* printf("%s superstring of %s\n", word, pat_list[i]); */
					if (pat_attr[i] != attribute) pat_attr[i] = 0;	/* union of two unequal attributes is all attributes */
					buffer = buffer_new;
					if (buffer_new >= buffer_end) break;
					continue;
				}
				/* Check if substring: delete all superstrings */
				for (i=0; i<num_pat; i++) {
					if (!is_mgrep_pat[i]) continue;
					substring = strstr(pat_list[i], word);
					if ((substring != NULL) && (substring[0] != '\0')) {	/* printf("%s substring of %s\n", word, pat_list[i]); */
						if (pat_attr[i] != attribute) attribute = 0;	/* union of two unequal attributes is all attributes */
						free(pat_list[i]);
						for (j=i; j<num_pat; j++) {
							pat_list[j] = pat_list[j+1];
							pat_lens[j] = pat_lens[j+1];
							is_mgrep_pat[j] = is_mgrep_pat[j+1];
							pat_attr[j] = pat_attr[j+1];
						}
						num_pat --;
						for (j=0; j<num_mgrep_pat; j++) {
							if (mgrep_pat_index[j] == i) {
								for (k=j; k<num_mgrep_pat; k++) {
									mgrep_pat_index[k] = mgrep_pat_index[k+1] - 1;
								}
								num_mgrep_pat --;
								break;
							}
						}
						i--;	/* to counter the ++ on top */
					}
				}
			}
#endif

                        buffer = buffer_new;
                        bp = (CHAR *) malloc(sizeof(CHAR) * word_length + 2);
                        if(bp == NULL) {
                                fprintf(stderr, "%s: malloc failed in %s:%d\n", GProgname, __FILE__, __LINE__);
                                return -1;
                        }
                        pat_list[num_pat] = bp;
                        pat_lens[num_pat] = word_length;
                        is_mgrep_pat[num_pat] = type;
                        pat_attr[num_pat] = attribute;
                        strcpy(pat_list[num_pat], word);
			pat_list[num_pat][word_length] = '\0';
			num_pat ++;
#if     BG_DEBUG
                        fprintf(debug, "word=%s\n", word);
#endif  /*BG_DEBUG*/
                        if(buffer_new >= buffer_end) break;
                        if(num_pat >= MAXNUM_PAT) {
                                fprintf(stderr, "%s: Warning! too many words in pattern (> %d): ignoring...\n", GProgname, MAXNUM_PAT);
                                break;
                        }
                }
	}
	for (i=0; i<num_pat; i++) {
		strcpy(&pat_buf[pat_ptr], pat_list[i]);
		pat_buf[pat_ptr + pat_lens[i]] = '\n';
		pat_buf[pat_ptr + pat_lens[i] + 1] = '\0';
		pat_ptr += (pat_lens[i] + 1);
	}
	return num_pat;
}