buildindex.cpp

WEB上实现的局域网FTP搜索程序

#include <dirent.h>
#include "conf.h"

void IndexFile(BTcnts fileno, char *filename)
{
    FILE *in;
    char buffer[MAX];
    long offset;
    char triple[KEYSIZE+1];
    BTreeItem *MyTriple;
    int j,len,k,valid;
    char *ptr;
    long triplecnt, ref;

    printf("Indexing %3d %40s... ", fileno, filename);
    fflush(stdout);
    triplecnt = 0L;
    if (!(in=fopen(filename,"r"))) 
    {
	perror("Cannot open file");
	return;
    }
    triple[KEYSIZE] = '\0';
    while (1)
    {
	offset = ftell(in);
	fgets(buffer, MAX, in);
	if (feof(in)) break;
	if ((ptr = strstr(buffer, "!!"))!=NULL) *ptr='\0';

	ptr = buffer + strlen(buffer);
	while (ptr!=buffer && *ptr!='/') ptr--;
	if (*ptr=='/') ptr++;

//	printf("processing... %s\n", ptr);

	len = strlen(ptr) - KEYSIZE;
	for (j = 0;j <= len; j++)
	{
	    strncpy(triple, &ptr[j], KEYSIZE);
	    MyTriple = LocateBItem(triple, Bhead);
	    if (!MyTriple) 
	    {
		printf("Error: Btree does not contain `%s' as found in\n", triple);
		printf("`%s'\n", buffer);
		continue;
	    }
	    if (MyTriple->lastfileno == fileno && MyTriple->lastoffset == offset) 
		continue;
	    if (MyTriple->lastfileno != fileno) 
	    {
		MyTriple->lastfileno = fileno;
		AddHostRef(MyTriple, fileno);
	    }
	    AddRef(MyTriple, offset);
	    MyTriple->lastoffset = offset;
	    triplecnt++;
	}
    }
    fclose(in);
    printf("%8ld\n", triplecnt);
}

void IndexData(void)
{
    DIR *dir;
    dirent *ptr;
    BTcnts fileno;
    FILE *filedb, *filelist;
    char host[64], fn[64];
    int c, on;

    fileno = 0;
    filelist = fopen(DBHOSTS, "r");
    while (!feof(filelist))
    {
	fscanf(filelist, "%d %d %s\n", &c, &on, host);
	sprintf(fn, "ftpdata/%s.dat", host);
	IndexFile(c, fn);
    }
}

int main(void)
{
    long lo;

    AnnounceMethod();
    /* Load triples from stdin */
    LoadTriples();
    printf("Found %ld unique keys\n",triplecount);

    /* Ok, create the tree */
    printf("Try create the btree..\n");
    CreateBtree(0L,triplecount);
    printf("Created...\n");
    for(lo=0L;lo<triplecount;lo++) 
	addB(TriIndex[lo],Bhead,&Bhead,&BTreeMax);
    printf("Created %ld Btree items\n",nodecount);

    /* Free the linked triple list as it is no longer needed */
    Tptr=Thead;
    while(Tptr) {Tptr2=Tptr->next; free(Tptr); Tptr=Tptr2;}

    /* Read the databases, index them with Btree... */
    IndexData();

    /* ... and save the index */
    diskoffset=0L;
    printf("Counting the host database size\n");
    IndexHostOffset(Bhead,&diskoffset);
    printf("Expected size of the index: %ld KB\n",
	   (long)(diskoffset/1024));
    SaveHostIndex(DBINDEX, DBITEMOFFSET);
    printf("Total %ld refs.\n", refcount);

    return(0);
}