wlistname.cpp

用来文本分类的

#include <memory.h>
#include <string.h>
#include "comm.h"
#include "stringop.h"
#include "wlistname.h"

WListName::WListName(char *file) 
{
   FILE *fin; 
   TextReadOpen(fin,file);
   for (wordNum=0;!feof(fin);) 
   {
      string text;
      if (fscanf(fin,"\n %[^\n]s",text)<1) break;
      if (text[0]!='#') wordNum++;
   }
   rewind(fin);
   lmNames=(char **)malloc(sizeof(char *)*wordNum);
   outNames=(char **)malloc(sizeof(char *)*wordNum);   

   for (int i=0;i<wordNum;) 
   {
      string text,lname,oname; 
      ASSERT2(fscanf(fin,"\n %[^\n]s",text)>0,"Unexpected EOF of file %s",file);

      if (text[0]=='#') continue;
      oname[0]='\0'; sscanf(text,"%s %[^\n]s",lname,oname);
      if (!oname[0]) strcpy(oname,lname);

      lmNames[i]=strdup(StringOP::ToLower(lname));
      outNames[i++]=strdup(StringOP::ToLower(oname));
   }
   fclose(fin);
   nameIdxHash=NULL;
}

void WListName::MakeLMNameIndex(void) 
{
   int i,wn4=wordNum*4;
   nameIdxHash=(int *)malloc(sizeof(int)*wn4);

   for (i=0;i<wn4;i++) nameIdxHash[i]= -1;

   for (i=0;i<wordNum;i++) 
   {
      int j=1,hash=0,k; 
	  char *wd=lmNames[i];
      for (k=strlen(wd)-1;k>=0;k--) 
		  hash=(hash<<3)+(int)wd[k];

      do 
	  {
         hash=(abs(hash+(j++))%wn4);
      }while (nameIdxHash[hash]>=0);

      nameIdxHash[hash]=i;
   }
}

int WListName::LMNameIdx(char *word) 
{
   int i,hash=0,j=1,wn4=wordNum*4;

   for (int k=strlen(word)-1;k>=0;k--) 
   {
        hash=(hash<<3)+(int)word[k];
   }

   do 
   {
      hash=(abs(hash+(j++))%wn4);
      if ((i=nameIdxHash[hash])<0) return -1;
   } while (strcmp(lmNames[i],StringOP::ToLower(word)));
   return i;
}

WListName::~WListName(void) 
{
	for (int i=0;i<wordNum;i++) 
	{
		free(lmNames[i]); free(outNames[i]);
	}

	free(lmNames); 
	free(outNames);
	if (nameIdxHash) free(nameIdxHash);
	
}

void WListName::WListNameConstruct(char *file)
{
	 FILE *fin; 
   TextReadOpen(fin,file);
   for (wordNum=0;!feof(fin);) 
   {
      string text;
      if (fscanf(fin,"\n %[^\n]s",text)<1) break;
      if (text[0]!='#') wordNum++;
   }
   rewind(fin);
   lmNames=(char **)malloc(sizeof(char *)*wordNum);
   outNames=(char **)malloc(sizeof(char *)*wordNum);   

   for (int i=0;i<wordNum;) 
   {
      string text,lname,oname; 
      ASSERT2(fscanf(fin,"\n %[^\n]s",text)>0,"Unexpected EOF of file %s",file);

      if (text[0]=='#') continue;
      oname[0]='\0'; sscanf(text,"%s %[^\n]s",lname,oname);
      if (!oname[0]) strcpy(oname,lname);

      lmNames[i]=strdup(StringOP::ToLower(lname));
      outNames[i++]=strdup(StringOP::ToLower(oname));
   }
   fclose(fin);
   nameIdxHash=NULL;
}