merge.c

harvest是一个下载html网页得机器人

** This function is used to remap file numbers from index
** files to a new merged index file.
*/

void remap(oldnum, newnum)
     int oldnum;
     int newnum;
{
	unsigned hashval;
	char tmpstr[MAXSTRLEN];
	struct mapentry *mp;

	mp = (struct mapentry *) emalloc(sizeof(struct mapentry));
	mp->oldnum = oldnum;
	mp->newnum = newnum;

	sprintf(tmpstr, "%d", oldnum);
	hashval = bighash(tmpstr);
	mp->next = mapentrylist[hashval];
	mapentrylist[hashval] = mp;
}


/* This retrieves the number associated with another.
*/


int getmap(num)
     int num;
{
	unsigned hashval;
	char tmpstr[MAXSTRLEN];
	struct mapentry *mp;

	sprintf(tmpstr, "%d", num);
	hashval = bighash(tmpstr);
	mp = mapentrylist[hashval];

	while (mp != NULL) {
		if (mp->oldnum == num)
			return mp->newnum;
		mp = mp->next;
	}
	return num;
}
#endif


/* This marks a number as having been printed.
*/

void marknum(num)
     int num;
{
	unsigned hashval;
	char tmpstr[MAXSTRLEN];
	struct markentry *mp;

	mp = (struct markentry *) emalloc(sizeof(struct markentry));
	mp->num = num;

	sprintf(tmpstr, "%d", num);
	hashval = bighash(tmpstr);
	mp->next = markentrylist[hashval];
	markentrylist[hashval] = mp;
}

#ifdef NDEF
/* Same thing but for merge only */
void marknumMerge(num, attribute)
     int num;
     int attribute;
{
        unsigned hashval;
        char tmpstr[MAXSTRLEN];
        struct markentryMerge *mp;
 
        mp = (struct markentryMerge *) emalloc(sizeof(struct markentryMerge));
        mp->num = num;
	mp->attribute = attribute;
 
        sprintf(tmpstr, "%d", num);
        hashval = bighash(tmpstr);
        mp->next = markentrylistMerge[hashval];
        markentrylistMerge[hashval] = mp;
}      
#endif

/* Has a number been printed?
*/

int ismarked(num)
     int num;
{
	unsigned hashval;
	char tmpstr[MAXSTRLEN];
	struct markentry *mp;

	sprintf(tmpstr, "%d", num);
	hashval = bighash(tmpstr);
	mp = markentrylist[hashval];

	while (mp != NULL) {
		if (mp->num == num)
			return 1;
		mp = mp->next;
	}
	return 0;
}

#ifdef NDEF
int ismarkedMerge(int num,int attribute)
{
        unsigned hashval;
        char tmpstr[MAXSTRLEN];
        struct markentryMerge *mp;
 
        sprintf(tmpstr, "%d", num);
        hashval = bighash(tmpstr);
        mp = markentrylistMerge[hashval];
 
        while (mp != NULL) {
                if ( (mp->num == num) && (mp->attribute == attribute) )
                        return 1;
                mp = mp->next;
	      }
        return 0;
      }
#endif

/* Initialize the marking list.
*/

void initmarkentrylist()
{
	int i;
	struct markentry *mp;

	for (i = 0; i < BIGHASHSIZE; i++) {
		mp = markentrylist[i];
		if (mp != NULL)
			free(mp);
		markentrylist[i] = NULL;
	}
}

#ifdef NDEF
void initmarkentrylistMerge()
{
        int i;
        struct markentryMerge *mp;
 
        for (i = 0; i < BIGHASHSIZE; i++) {
                mp = markentrylistMerge[i];
                if (mp != NULL)
                        free(mp);
                markentrylistMerge[i] = NULL;
	      }
}      


/* Initialize the main file list.
*/

void initindexfilehashlist()
{
  int i;
  struct indexfileinfo *ip;
  
  for (i = 0; i < BIGHASHSIZE; i++) {
    ip = indexfilehashlist[i];
    if (ip != NULL)
      free(ip);
    indexfilehashlist[i] = NULL;
  }
}

/* Initialize the mapentrylist 
*/

void initmapentrylist()
{
  int i;
  struct mapentry *ip;
  
  for (i = 0; i < BIGHASHSIZE; i++) {
    ip = mapentrylist[i];
    if (ip != NULL)
      free(ip);
    mapentrylist[i] = NULL;
  }
}

#endif

/* Frees up used index entries, my best attempt at memory management...
** I still have bytes leaking elsewhere...
*/

void freeindexentry(ip)
     struct indexentry *ip;
{
	struct resultMerge *rp, *oldp;

	free(ip->word);
	rp = ip->result;
	while (rp != NULL) {
		oldp = rp;
		rp = rp->next;
		free(oldp);
	}
	free(ip);
}

/* Translates a file number into something that can be compressed.
*/

int encodefilenum(num)
     int num;
{
	int i, j;

	for (i = j = 0; i != num; i++) {
		j++;
		if (!(j % 128))
			j++;
	}
	return j;
}

/* Translates a compressed file number into a correct file number.
*/

int decodefilenum(num)
     int num;
{
        int i, extra;

        for (i = 1, extra = 0; i < num; i++)
                if (!(i % 128)) {
                        extra++;
                        i++;
                }
        num -= extra;

	return num;
}

/* Similar to addtoresultlist, but also adding the meta name
*/

struct resultMerge *addtoresultlistMerge(struct resultMerge *rp, 
					 int filenum, int rank,int attribute)
{
        struct resultMerge *newnode;
        static struct resultMerge *head;
 
        newnode = (struct resultMerge *) emalloc(sizeof(struct resultMerge));
        newnode->filenum = filenum;
        newnode->rank = rank;
	newnode->attribute = attribute;
        newnode->next = NULL;
 
        if (rp == NULL)
                rp = newnode;
        else
                head->next = newnode;
 
        head = newnode;
 
        return rp;
}

/* Reads the meta names from the index. Needs to be different from
** readMetaNames because needs to zero out the counter.
*/
struct metaMergeEntry* readMergeMeta(metaFile,fp)
     struct metaMergeEntry* metaFile;
     FILE* fp;
{     
  int i, c, counter;
  char word[MAXWORDLEN];
  
  counter = 0;
  fseek(fp, offsets[METANAMEPOS], 0);
  for (i = 0; (c = fgetc(fp)) != '\n' && c != EOF; ){
    if (!isspace(c))
      word[i++] = c;
    else {
      word[i] = '\0';
      metaFile = addMetaMerge(metaFile, word,&counter);
      i = 0;
    }
  }
  return metaFile;
}

/* Adds an entry to the list of meta names for one index, 
** setting the new index to 0 - it will then be set by 
** createMetaMerge.
*/

struct metaMergeEntry* addMetaMerge(metaFile, metaWord,counter)
     struct metaMergeEntry* metaFile;
     char* metaWord;
     int* counter;
{
  int i;
  struct metaMergeEntry* newEntry;
  struct metaMergeEntry* tmpEntry;
  
  if (*counter == 0)
    *counter = 2;
  else if ((*counter) == 1 ||  (!((*counter) % 128)) )
   (*counter)++;
  for( i=0; metaWord[i]; i++)
    metaWord[i] =  tolower(metaWord[i]);
  newEntry = (struct metaMergeEntry*) emalloc(sizeof(struct metaMergeEntry));
  newEntry->metaName = (char*)mystrdup(metaWord);
  newEntry->oldIndex = (*counter)++;
  newEntry->newIndex = 0;
  newEntry->next = NULL;
  if (metaFile)
    {
      for(tmpEntry=metaFile;tmpEntry->next!=NULL;tmpEntry=tmpEntry->next)
        ;
      tmpEntry->next = newEntry;
    }
  else
    metaFile = newEntry;
  
  return metaFile;
}

/* Creates a list of all the meta names in the indexes
*/
struct metaEntry* createMetaMerge(metaFile1, metaFile2)
     struct metaMergeEntry* metaFile1;
     struct metaMergeEntry* metaFile2;
{
  struct metaMergeEntry* tmpEntry;
  int counter;

  metaEntryList = NULL;
  counter = 0;
  for (tmpEntry=metaFile1;tmpEntry;tmpEntry=tmpEntry->next)
    metaEntryList = addMetaMergeList(metaEntryList,tmpEntry,&counter);
  
  for (tmpEntry=metaFile2;tmpEntry;tmpEntry=tmpEntry->next)
    metaEntryList = addMetaMergeList(metaEntryList,tmpEntry,&counter);

  return metaEntryList;
}

/* Adds an entry to the merged meta names list and changes the
 ** new index in the idividual file entry
 */

struct metaEntry* addMetaMergeList(metaEntryList,metaFileEntry,count)
     struct metaEntry* metaEntryList;
     struct metaMergeEntry* metaFileEntry;
     int* count;
{
  int i, wordExists, newIndex;
  struct metaEntry* newEntry;
  struct metaEntry* tmpEntry;
  struct metaEntry* last;
  char *metaWord, *compWord;
  
  wordExists = 0;
  if ((*count) == 0)
    *count = 2;
  else if ((*count) == 1 ||  (!((*count) % 128)) )
    (*count)++;
  metaWord = metaFileEntry->metaName;
  
  for( i=0; metaWord[i]; i++)
    metaWord[i] =  tolower(metaWord[i]);
  if (metaEntryList)
    {
      for(tmpEntry=metaEntryList;tmpEntry;tmpEntry=tmpEntry->next)
	{
	  if (tmpEntry->next == NULL)
	    last = tmpEntry;
	  compWord = tmpEntry->metaName;
	  if (!strcmp(compWord,metaWord) ) 
	    {
	      wordExists = 1;
	      newIndex = tmpEntry->index;
	      break;
	    }
	}
      if (wordExists)
	metaFileEntry->newIndex = newIndex;
      else {
	newEntry = (struct metaEntry*) emalloc(sizeof(struct metaEntry));
	newEntry->metaName = (char*)mystrdup(metaWord);
	newEntry->index = *count;
	newEntry->next = NULL;
	metaFileEntry->newIndex = (*count)++;
	last->next = newEntry;
      }
    }
  else {
    newEntry = (struct metaEntry*) emalloc(sizeof(struct metaEntry));
    newEntry->metaName = (char*)mystrdup(metaWord);
    newEntry->index = *count;
    newEntry->next = NULL;
    metaFileEntry->newIndex = (*count)++;
    metaEntryList = newEntry;
  }
  return metaEntryList;
}