cluster.c

隐马尔科夫模型工具箱

   }
}
#endif


/* Complain about a broken header */
static void invalid_header(void)
{
   HError(17013, "Classmap has broken header - missing '='");
}


/* Import a HLM classmap file. Currently ignores contents of 'EscMode' field. GLM Also non-IN?*/
void import_classmap(char *fname, int numb_words)
{
#define max_line_len 500 
   FILE     *file;               /* Input file handle */
   char      line[max_line_len]; /* Line read buffer */
   int       C;                  /* Current class index */
   int       size;               /* Size of current class */
   int       i;                  /* Loop counter */
   char     *ptr;                /* Text pointer */
   UInt      id;                 /* Word id */
   int       reassigned = 0;     /* Number of reassigned classes */
   int       unexpected = 0;     /* Number of unexpected lines trailing class descriptions */
   Boolean   pipe_status;

   if (trace & T_FILE) {
      printf("Importing classmap '%s'\n", fname);
   }

   W = numb_words;
   clMemb = CNew(&global_stack, W * sizeof(int));
  
   /* Set impossible classmap in order to do integrity check after import */
   for (i=0; i<W; i++) {
      clMemb[i] = -1;
   }
   
   N = 0;
   file = FOpen(fname, LCMapFilter, &pipe_status);
   check_file(file, fname, "import_classmap");
   while (fgets(line, max_line_len, file)) {
      if (strncmp(line, "Entries", 7)==0) {
         ptr = strchr(line, '=');
         if (!ptr) invalid_header();
         ptr++;
         ptr = strtok(ptr, " \t\n");
         N = atoi(ptr);
         if (trace & T_EXTRA) {
            printf("Number of classes = %d\n", N);
         }
      }
      else if (strncmp(line, "Iterations", 10)==0) {
         ptr = strchr(line, '=');
         if (!ptr) invalid_header();
         ptr++;
         ptr = strtok(ptr, " \t\n");
         export_index = atoi(ptr);
      }
      else if (strncmp(line, "EscMode", 7)==0) {
         ptr = strchr(line, '=');
         if (!ptr) invalid_header();
         ptr++;
         ptr = strtok(ptr, " \t\n");
         if (strcmp(ptr, "HTK")==0) {
            if (inCMapRawTrap && inCMapRaw) {
               HError(-17013, "Class map specifies HTK escaping on input but configuration file specifies Raw escaping -- using HTK escaping for input");
            }
            inCMapRaw = FALSE;
         }
         else if (strcmp(ptr, "Raw")==0) {
            if (inCMapRawTrap && !inCMapRaw) {
               HError(-17013, "Class map specifies Raw escaping on input but configuration file specifies HTK escaping -- using Raw escaping for input");
            }
            inCMapRaw = TRUE;
         }
         else {
            HError(17013, "Classmap has unknown escaping of type '%s'", ptr);
         }
         if (!outCMapRawTrap) {
            if (outCMapRaw != inCMapRaw) {
               HError(-17013, "Setting output class map escaping to same format as input class map (%s)", inCMapRaw?"Raw":"HTK");
            }
            outCMapRaw = inCMapRaw;  /* This is common sense */
         }
         if (inCMapRaw != outCMapRaw) {
            HError(-17013, "Input class map escaping and output class map escaping differ (this is not a problem -- this warning is to alert you in case you meant them to be the same)");
         }
      }
      else if (strncmp(line, "\\Classes\\", 9)==0) {
         break;
      }
   }
   if (feof(file)) {
      HError(17013, "Classmap file is corrupt/contains no classes!");
   }

   if (!N) {
      HError(17013, "Corrupt classmap header - must specify number of classes!");
   }

   if (trace & T_EXTRA) {
      printf("Iterations = %d\n", export_index);
   }

   C = 0;
   while (fgets(line, max_line_len, file)) {
      if (C>=N) {
         if (strstr(line, " IN")) {
            HError(17013, "More classes are described than are specified in the header!");
         }
         else {
            ptr = strtok(line, " \t\n");
            if (!ptr)
               continue;
            else {
               HError(-17013, "Warning: ignoring '%s' at end of classmap file", ptr);
            }
         }
      }

      if (strstr(line, " IN")) {
         /* Start of a new class */
         /* Make this class 'C' */
         strtok(line, " \t");
         ptr = strtok(NULL, " \t");
         if (!ptr) {
            HError(17013, "Failure reading class header %d in classmap (no id)", C);
         }
         if (atoi(ptr) != C+1) {
            /* We'll renumber this class */
            reassigned++;
         }
         ptr = strtok(NULL, " \t");
         if (!ptr) {
            HError(17013, "Failure reading class header %d in classmap (no size)", C);
         }
         size = atoi(ptr); /* Read number of words in class */

         for (i=0; i<size; i++) {
            fgets(line, max_line_len, file);
            ptr = strtok(line, " \t\n");
            if (!ptr) {
               /* Warn about the blank line */
               HError(-17013, "Found empty line inside class %d definition", C);
               i--;
               continue;
            }
            /* Unescape word if necessary */
            if (!inCMapRaw) {
               if (strlen(ptr)>255) {
                  HError(17013, "Cannot handle words longer than 255 characters when using HTK escaping (recompile with higher tmp[] buffer size in Cluster.c)");
               }
               ParseString(ptr, tmp);

               /* Put word in class */
               id = get_id_from_word(tmp);
            }
            else {
               /* Put word in class */
               id = get_id_from_word(ptr);
            }

            if (inCMapRaw && strcmp(what_is_word(id), ptr)!=0) {
               HError(17095, "import_classmap: word '%s' is id '%d'; id is '%s'!", ptr, id, what_is_word(id));
            }
            else if (!inCMapRaw && strcmp(what_is_word(id), tmp)!=0) {
               HError(17095, "import_classmap: word '%s' is id '%d'; id is '%s'!", tmp, id, what_is_word(id));
            }

            if (clMemb[id] != -1) {
               HError(17094, "Word '%s' occurs more than once in classmap!", ptr);
            }
            clMemb[id] = C;
         }
         C++;
      }
      else {
         /* Where is class header? It's gone missing! */
         if (strlen(line)>0) {
            if (strchr(line, '\n'))
               *strchr(line, '\n')='\0'; /* Strip linefeed */
            if (strlen(line)==0)
               continue;
            HError(-17013, "Unexpected line '%s' in classmap", line);
            unexpected++;
         }
         if (unexpected>9) {
            HError(17013, "Too many unexpected lines in classmap - aborting now");
         }
         /* Loop round to see if it's coming up next */
      }
   }

   if (C<N) {
      HError(17013, "Less classes are described than are specified in the header!");
   }

   if (trace & T_TOP) {
      if (reassigned) {
         if (reassigned>1) {
            printf("%d class ids were reassigned\n", reassigned);
         }
         else {
            printf("1 class id was reassigned\n");
         }
      }
      else {
         printf("No class ids were reassigned\n");
      }
   }

   /* Check all words were assigned */
   for (i=0; i<W; i++) {
      if (clMemb[i]==-1) {
         HError(17052,"import_classmap: Not all words were assigned to classes");
      }
   }

   FClose(file, pipe_status);
   if (trace & T_FILE) {
      printf("Class map import successful\n");
   }
}


/* Write out p(word | class) probabilities */
void write_word_probs(char *filename)
{
   FILE *out;
   int   i;   /* Loop counter */
   double probability;
   Boolean pipe_status;

   /* These files never use HTK escaping */
   
   out = FOpen(filename, NoOFilter, &pipe_status);
   check_file(out, filename, "write_word_probs");

   /* Write header */
   fprintf(out, "Word|Class probabilities\n");
   fprintf(out, "\n");
   fprintf(out, "Derived from: %s\n", export_prefix);
   fprintf(out, "Number of classes: %d\n", N);
   fprintf(out, "Number of words: %d\n", W);
   fprintf(out, "Iterations: %d\n", export_index);
   fprintf(out, "\n");
   fprintf(out, "%-15s\tClass name\tProbability (log)\n", "Word");

   for (i=0; i<W; i++) {
      if (uni[i]==0) uni[i]=1;
   }

   /* Use tmp_sum1[] to save having to allocate a new array (so can't call
      this from within a class change calculation, but this isn't a problem!)   */
   for (i=0; i<N; i++) {
      tmp_sum1[i] = 0;
   }
   for (i=0; i<W; i++) {
      tmp_sum1[clMemb[i]] += uni[i];
   }

   for (i=0; i<W; i++) {
      probability = (double)uni[i]/((double)tmp_sum1[clMemb[i]]);

      fprintf(out, "%-15s\tCLASS%-4d\t%f\n", what_is_word(i), clMemb[i]+1,
              LOG_NATURAL(probability));

      if (LOG_NATURAL(probability)<-90) {
         printf("prob is %f, discount is %f, uni is %d\n", LOG_NATURAL((double)uni[i]/((double)tmp_sum1[clMemb[i]])), mlv[clMemb[i]], uni[i]);
      }
   }
   FClose(out, pipe_status);
   if (trace & T_FILE) {
      printf("Wrote word|class probabilities to '%s'\n", filename);
   }
}


/* Write out p(word | class) counts */
void write_word_counts(char *filename)
{
   FILE *out;
   int   i;   /* Loop counter */
   Boolean pipe_status;

   /* These files never use HTK escaping */

   /* Open output file */
   out = FOpen(filename, NoOFilter, &pipe_status);
   check_file(out, filename, "write_word_counts");

   /* Write header */
   fprintf(out, "Word|Class counts\n");
   fprintf(out, "\n");
   fprintf(out, "Derived from: %s\n", export_prefix);
   fprintf(out, "Number of classes: %d\n", N);
   fprintf(out, "Number of words: %d\n", W);
   fprintf(out, "Iterations: %d\n", export_index);
   fprintf(out, "\n");
   fprintf(out, "%-15s\tClass name\tCount\n", "Word");
   
   for (i=0; i<W; i++) {
      fprintf(out, "%-15s\tCLASS%-4d\t%d\n", what_is_word(i), clMemb[i]+1, uni[i]);
   }

   FClose(out, pipe_status);
   if (trace & T_FILE) {
      printf("Wrote word|class counts to '%s'\n", filename);
   }
}


/* Specify whether to keep the unknown word in its own solo-member class or not */
void classes_keep_unk_separate(int keep_separate)
{
   unk_sep = (Boolean) keep_separate;
   start_class = unk_sep?3:2;
}


/* Pass in start, end and unknown word ids */
void set_ids(int start, int end, int unk)
{
   start_id = start;
   end_id = end;
   unk_id = unk;
}


/* This set of functions takes unigram counts, stores them, and
   then allows them to be retrieved. It simply allocates a count
   for each possible word id, since they are allocated in a
   continuous block.
*/

/* Initialise this unigram storage module */
void unigram_init(int words)
{
   max_words = words;
   uni = CNew(&global_stack, words * sizeof(unigram));
   sum_of_all_uni_counts = 0;
}


/* Add a unigram */
void unigram_add(NGram ng, int count)
{
   if (ng[0]>=max_words) {
      /* Something's gone wrong */
      HError(17093, "unigram_add: Found a word id higher than the base+number of words - word ids are expected to be allocated in an unbroken chunk\n[Current unigram is (%d); number of words is %d]", ng[0], max_words);
   }
   uni[ng[0]] += count;
   sum_of_all_uni_counts += count;
   return;
}


/* Read a unigram */
UInt unigram_read(UInt id)
{
#ifdef INTEGRITY_CHECK
      if ((id<0) || (id>=max_words)) {
         HError(17092, "unigram_read: attempt to read unigram outside bounds (%d; %d words)", id, max_words);
      }
#endif
   return uni[id];
}


/* This section contains functions to store a sequence of bigrams - they must
   be sequenced before passing to this code, since it relies on the input
   being sorted.
   Both forward and backward word to all bigrams look-up tables are built,
   so given either u or v from a bigram (u,v) then the set of all (u,*) or
   (*,v) can be found. */

/* Grab some space from our current local storage block */
static void *get_space(int size)
{
   static void* ptr;

   /* Test against our not-worth-using cut-off point */
   if (size>block_cut_off)
      return New(&global_stack, size);

   /* Use New() again if necessary to get a new block */
   if (((int)block+(int)size) >= (int)block_end) {
      block = New(&global_heap, block_grab_size);
      block_end = (void *) ((int)block+(int)block_grab_size);
   }

   /* Hand back the next free space */
   ptr = block;
   block = (void*) ((int) block + (int) size);     /* Next free byte */
   block = (void*) ((((int)block)+3) & (~(int)3)); /* Word-align */

   return ptr;
}


/* Add a bigram */
void bigram_add(NGram ng, int count)