list.h

网页抓取程序

/* OpenWebSpider
*
 *  Authors:     Stefano Alimonti AND Stefano Fantin
 *  Version:     0.7
 *  E-Mails:     shen139 [at] openwebspider (dot) org AND [e-mail]
*
*
* This file is part of OpenWebSpider
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*
*/

#ifndef __LIST
#define __LIST

int lstInited = 0;

int lstFreeAll(NODE* first)
{
	NODE* cur = first;
	NODE* tmp;
	lstInited = 0;
	
	lstInited=0;
	
	if(first==NULL)
		return 1;
	
	if(cur->field == NULL && cur->next == NULL)
	{
		return 1;
	}
	
	while(cur != NULL)
	{
		tmp=cur->next;
		
		if(cur->field!=NULL)
			free(cur->field);
		
		if(cur!=NULL)
			free(cur);
		
		//FREE(cur->field);
		//FREE(cur);
		cur=tmp;
	}
	
	return 1;
}

NODE* lstGetNodeX(NODE* first, int x)
{
	NODE* cur = first;
	
	thrdBlock(BLOCKTHRDHST);
	while(cur != NULL && x>0)	//to correct in do while
	{
		x--;
		cur=cur->next;
	}
	thrdUnBlock(BLOCKTHRDHST);
	
	return (x==0) ? cur : NULL;
}

NODE* lstGetLastNode(NODE* first, int* NumOfNodes)
{
	NODE* cur = first;
	
	if(lstInited == 0)
		return NULL;
	
	if(NumOfNodes!=NULL)
		*NumOfNodes=0;
	
	while(cur->next != NULL)
	{
		if(NumOfNodes!=NULL)
			(*NumOfNodes)++;
		
		cur=cur->next;
	}
	
	return cur;
}

/********************************************************************************************/

INVERTED_INDEX* InitII()
{
	INVERTED_INDEX* first=malloc(sizeof(INVERTED_INDEX));
	
	first->doc_id = 0;
	first->position = 0;
	first->next = NULL;
	first->last = first;
	
	return first;
}


OOI_NODE* InitLexicon()
{
	OOI_NODE* first=malloc(sizeof(OOI_NODE)*LEXICONWORDSIZE);
	
	memset(first,0,sizeof(OOI_NODE)*LEXICONWORDSIZE);

	lexicon_number_of_elements = 0;
	lexicon_actual_size = LEXICONWORDSIZE;
	
	return first;
}

#define swap(n1, n2)			\
{								\
	OOI_NODE tmp;				\
								\
	tmp.id = n1.id;				\
	tmp.field = n1.field;		\
	tmp.ii = n1.ii;				\
								\
	n1.id = n2.id;				\
	n1.field = n2.field;		\
	n1.ii = n2.ii;				\
								\
	n2.id = tmp.id;				\
	n2.field = tmp.field;		\
	n2.ii = tmp.ii;				\
								\
}								\
	
void insertion_sort(OOI_NODE* lexicon, int n) 
{
    int i, j;
	char* app;
	
    for (i=1; i<n; i++) {
        app = lexicon[i].field;
		
        j = i-1;
        while ((j>=0) && (strcmp(lexicon[j].field, app)>0)) 
		{
			swap(lexicon[j+1],lexicon[j]);
            j--;
        }
		
        lexicon[j+1].field = app;
    }
	
}

/*
int compare(char* p1, char* p2)
{
return strcmp (p1, p2);
}

int comp(OOI_NODE * a, OOI_NODE* b)
{
return strcmp(a->field,b->field);
}
*/

int lstAddWord(OOI_NODE** lexicon,char* word)
{
	OOI_NODE* cur;
	
	if(lexicon_number_of_elements+1 >= lexicon_actual_size)
	{
		lexicon_actual_size = lexicon_actual_size + 100;
		*lexicon = realloc((OOI_NODE*)*lexicon, sizeof(OOI_NODE)*lexicon_actual_size);
	}
	
	cur = *lexicon;
	
	cur[lexicon_number_of_elements].id = lexicon_number_of_elements+1;
	cur[lexicon_number_of_elements].field = (char*)malloc(strlen(word)+1);
	strcpy(cur[lexicon_number_of_elements].field, word);
	
	cur[lexicon_number_of_elements].ii = InitII();
	
	lexicon_number_of_elements++;
	
	insertion_sort (*lexicon, lexicon_number_of_elements);
	
	/* qsort(*lexicon, number_of_elements,sizeof(OOI_NODE),comp); */
	
	return 1;
}

int ndzLookForWord(OOI_NODE* lexicon, char* word)
{
	unsigned int m;
	int p, u;
	p = 0;
	u = lexicon_number_of_elements-1;
	while(p<u)
	{
		m = (p+u)/2;
		if(lexicon[m].id>0 &&
			strcmp(lexicon[m].field,word)<0)
			p = m+1;
		else
			u = m;
	}
	
	if(u>=0 && lexicon[u].id>0 &&
		strcmp(lexicon[u].field,word)==0)
		return u;
	else
		return -1;
	
}

void FreeOwsIndex(OOI_NODE* lexicon)
{
	unsigned int i;
	INVERTED_INDEX* ii;
	INVERTED_INDEX* cur;
	
	for(i=0; i < lexicon_number_of_elements; i++)
	{
		ii = lexicon[i].ii;
		while(ii != NULL)
		{
			cur = ii->next;
			FREE(ii)
			ii = cur;
		}
	}

	FREE(lexicon)

}

#endif

/*EOF*/