hlabel.c

隐马尔科夫模型工具箱

/* ----------------------------------------------------------- */
/*                                                             */
/*                          ___                                */
/*                       |_| | |_/   SPEECH                    */
/*                       | | | | \   RECOGNITION               */
/*                       =========   SOFTWARE                  */ 
/*                                                             */
/*                                                             */
/* ----------------------------------------------------------- */
/*         Copyright: Microsoft Corporation                    */
/*          1995-2000 Redmond, Washington USA                  */
/*                    http://www.microsoft.com                 */
/*                                                             */
/*   Use of this software is governed by a License Agreement   */
/*    ** See the file License for the Conditions of Use  **    */
/*    **     This banner notice must not be removed      **    */
/*                                                             */
/* ----------------------------------------------------------- */
/*         File: HLabel.c:   Speech Label File Input           */
/* ----------------------------------------------------------- */

char *hlabel_version = "!HVER!HLabel:   3.2 [CUED 09/12/02]";
char *hlabel_vc_id = "$Id: HLabel.c,v 1.10 2002/12/19 16:37:11 ge204 Exp $";

#include "HShell.h"
#include "HMem.h"
#include "HMath.h"
#include "HWave.h"
#include "HLabel.h"

/* ----------------------------- Trace Flags ------------------------- */

static int trace = 0;
#define T_MLF     0001     /* Top Level MLF tracing */
#define T_MHASH   0002     /* MLF Hashing */
#define T_MAT     0004     /* MLF Pattern Matching */
#define T_SUBD    0010     /* MLF Subdir search */
#define T_HTKL    0020     /* HTK Lab file loading */
#define T_HASH    0040     /* GetLabId Hashing */
#define T_SAV     0100     /* Label file saving */

/* -------  Input file formats supported by this module ---------------
 
      TIMIT - text files with each label on a line in the form
                  start-sample end-sample  label   
              (eg 456 7899 sh). 
              
      SCRIBE - SAM format basically a sequence of tagged lines of
              form
                       <header stuff>
                   LBD:
                   LBB:  st, ,en, label
                   LBB:  st, ,en, label
                       ....
                   ELF:
                   

      HTK   - a HTK transcription consists of one or more label lists
              where each label list is separated by 3 / characters
                 <transcription> = <label list> {"///" <label list>}
              each label list is a sequence of lines of text, each defining
              a labelled segment of speech
                 <label list> = <labelseg> { <labelseg> }
              each label seg consists of optional start and end times in
              units of 100ns followed by one or more label names followed 
         by an optional score  
        <label> = [<start> <end>] < <name> [<score>] > [";" <comment>]
              The start and end times are written as ints externally but
              are stored as doubles, the score can be
              int or float.  The name must begin with a non-digit
              character other than the level separator.

      ESPS  - waves label files. Text files with each label on a line 
              in the form
                 time color-code label   (eg 0.138375 121 h#)
              see the ESPS waves manual pages for details

*/

/* ----------------- Master Label File Data Structures ---------

An MLF is a file containing a sequence of patterns, each pattern
either points to a subdirectory to search for a label file or
has a definition immediately following it (terminated by a period
on a single line.  The file must start with the MLF id.
   
#!MLF!#
"pattern1" -> "subdir1"
"pattern2" => "subdir2"
"pattern3"
0   100 sil
101 205 bah
206 300 sil
.
"pattern4"
etc
   
where -> denotes a simple search ie the name of the file matching
pattern1 must be in subdir1; => denotes a full search so that some
part of the file's path matching pattern2 must be in subdir2
*/

static ConfParam *cParm[MAXGLOBS];        /* config parameters */
static int numParm = 0;
static Boolean stripTriPhones = FALSE;   /* Enable triPhone stripping */
static int transLev = 0;           /* if >0 filter all but specified level */
static int transAlt = 0;           /* if >0 filter all but specified alt */
static Boolean compatMode = FALSE;  /* Allow spaces around . or /// */
static char labelQuote = 0;        /* How do we quote label names */

/* --------------- Global MLF Data Structures  --------- */

#define MLFCHUNKSIZE 500
#define MAXMLFS 200

static int      numMLFs = 0;     /* number of MLF files opened */
static FILE   * mlfile[MAXMLFS]; /* array [0..numMLFs-1] of MLF file */
static int      mlfUsed = 0;     /* number of entries in mlfTab */
static MLFEntry *mlfHead = NULL; /* head of linked list of MLFEntry */
static MLFEntry *mlfTail = NULL; /* tail of linked list of MLFEntry */
static MemHeap mlfHeap;          /* memory heap for MLF stuff */

typedef struct {
   FILE *file;
   LabId name;
} OutMLFEntry;

static FILE *outMLF = NULL;                 /* output MLF file, if any */ 
static int numOutMLF = 0;                   /* number of output MLFs */ 
static OutMLFEntry outMLFSet[MAXMLFS];      /* array of output MLFs */

/* ---------------- Label Name Hashing ----------------- */

#define HASHSIZE 5701                /* size of hash table */
static NameCell *hashtab[HASHSIZE];  /* the actual table */
static MemHeap namecellHeap;         /* heap for name cells */
static long numAccesses = 0;
static long numTests = 0;

/* Hash: return a hash value for given label name */
static unsigned Hash(char *name)
{
   unsigned hashval;

   for (hashval=0; *name != '\0'; name++)
      hashval = *name + 31*hashval;
   return hashval%HASHSIZE;
}

/* NewCell: return a pointer to a new NameCell */
static NameCell *NewCell(char *name)
{
   char *s;
   NameCell *p;
   int len;

   len = strlen(name);
   s = (char *)New(&namecellHeap,len+1);
   strcpy(s,name);
   p = (NameCell *) New(&namecellHeap,sizeof(NameCell));
   p->name = s; p->next = NULL; p->aux = NULL;
   return p;
}

/* EXPORT->InitLabel: initialise module */
void InitLabel(void)
{
   int i;
   Boolean b;
   char str[MAXSTRLEN];

   Register(hlabel_version,hlabel_vc_id);
   CreateHeap(&namecellHeap,"namecellHeap",MSTAK,1,0.5,5000,20000);
   for (i=0;i<HASHSIZE;i++)
      hashtab[i] = NULL;
   CreateHeap(&mlfHeap,"mlfHeap",MSTAK,1,0.5,10000,50000);
   numParm = GetConfig("HLABEL", TRUE, cParm, MAXGLOBS);
   if (numParm>0){
      if (GetConfInt(cParm,numParm,"TRACE",&i)) trace = i;
      if (GetConfBool(cParm,numParm,"STRIPTRIPHONES",&b)) 
         stripTriPhones = b;
      if (GetConfBool(cParm,numParm,"V1COMPAT",&b))  compatMode = b;
      if (GetConfStr(cParm,numParm,"LABELSQUOTE",str))
         labelQuote=str[0];
      if (GetConfInt(cParm,numParm,"TRANSALT",&i)) transAlt = i;
      if (GetConfInt(cParm,numParm,"TRANSLEV",&i)) transLev = i;
   }
}


/* EXPORT->GetLabId: return id of given name */
LabId GetLabId(char *name, Boolean insert)
{
   int h;
   NameCell *p;

   ++numAccesses; ++numTests;
   if ((trace&T_HASH) && numAccesses%100 == 0) 
      PrintNameTabStats();
   h = Hash(name); p = hashtab[h];
   if (p==NULL) {  /* special case - this slot empty */
      if (insert)
         p=hashtab[h]=NewCell(name);
      return p;
   }
   do{             /* general case - look for name */
      if (strcmp(name,p->name) == 0)
         return p; /* found it */
      ++numTests;
      p = p->next;
   } while (p != NULL);
   if (insert){    /* name not stored */
      p = NewCell(name);
      p->next = hashtab[h];
      hashtab[h] = p; 
   }
   return p;
}

/* EXPORT->PrintNameTabStats: print out statistics on hash table usage */
void PrintNameTabStats(void)
{
   printf("Name Table Statistics:\n");
   printf("Total Accesses: %ld\n", numAccesses);
   printf("Ave Search Len: %f\n",(float)numTests/(float)numAccesses); 
   PrintHeapStats(&namecellHeap);
   printf("\n"); fflush(stdout);
}

/* EXPORT->ReadLabel: into buf from file f and return TRUE if ok */
Boolean ReadLabel(FILE *f, char *buf)
{
   int c;
   
   c = fgetc(f);
   while (isspace(c)) c = fgetc(f);
   if (c==EOF || !isgraph(c)) return FALSE;
   do {
      *buf++ = c; c = fgetc(f);
   } while ( !isspace(c) && c != EOF);
   if (c != EOF) ungetc(c,f);
   *buf = '\0';
   return TRUE;
}

/* -------------------- Label List Handling -------------------- */

/* EXPORT->CreateTranscription: create empty transcription */
Transcription *CreateTranscription(MemHeap *x)
{
   Transcription *t;
   
   t = (Transcription *)New(x,sizeof(Transcription));
   t->head = t->tail = NULL;
   t->numLists = 0;
   return t;
}

/* EXPORT->CopyTranscription: create copy of given transcription */
Transcription *CopyTranscription(MemHeap *x, Transcription *t)
{
   Transcription *newt;
   LabList *ll,*newll;
   
   newt = CreateTranscription(x);
   ll = t->head;
   for (ll = t->head; ll != NULL; ll=ll->next){
      newll = CopyLabelList(x,ll);
      AddLabelList(newll,newt);
   }
   return newt;  
}

/* EXPORT->CreateLabelList: create a new label list with sentinels */
LabList* CreateLabelList(MemHeap *x, int maxAuxLab)
{
   LLink st,en;
   LabList *ll;
   
   ll = (LabList *)New(x,sizeof(LabList));
   st = (LLink)New(x,sizeof(Label));
   en = (LLink)New(x,sizeof(Label));
   st->labid = en->labid = NULL; 
   st->pred = NULL; st->succ = en;
   en->succ = NULL; en->pred = st;
   ll->head = st; ll->tail = en; ll->next = NULL;
   ll->maxAuxLab = maxAuxLab;
   return ll;
}

/* EXPORT->AddLabelList: Add given label list to transcription t */
void AddLabelList(LabList *ll, Transcription *t)
{
   if (ll==NULL) return;
   ++t->numLists;
   if (t->tail==NULL)
      t->head = t->tail = ll;
   else {
      t->tail->next = ll; t->tail = ll;
   }
}

/* EXPORT->GetLabelList: get n'th label list (n=1,2,...) */
LabList* GetLabelList(Transcription *t, int n)
{
   LabList* q;
   int i;
   
   if (n>t->numLists)
      HError(6570,"GetLabelList: n[%d] > numLists[%d]",n,t->numLists);
   q = t->head;
   for (i=1; i<n; i++) q = q->next;
   return q;
}

/* EXPORT->CopyLabelList: return a copy of given label list */
LabList* CopyLabelList(MemHeap *x, LabList* ll)
{
   LabList *newll;
   LLink p,q;
   
   newll = CreateLabelList(x,ll->maxAuxLab);
   for (q=ll->head->succ; q->succ != NULL; q = q->succ){
      p = AddLabel(x,newll,q->labid,q->start,q->end,q->score);
      if (ll->maxAuxLab > 0)
         AddAuxLab(p,ll->maxAuxLab,q->auxLab,q->auxScore);
   }
   return newll;
}

/* EXPORT->CreateLabel: create a label with maxAux aux slots */
LLink CreateLabel(MemHeap *x, int maxAux)
{
   LLink p;
   int i;
   LabId *id;
   float *s;
   
   p = (LLink)New(x,sizeof(Label));
   p->labid = NULL; p->score = 0.0; 
   p->auxLab = NULL; p->auxScore = NULL;
   p->start = p->end = 0;
   p->succ = p->pred = NULL;
   if (maxAux > 0) {
      id = (LabId *)New(x,sizeof(LabId)*maxAux);
      s = (float *)New(x,sizeof(float)*maxAux);
      p->auxLab = id - 1; p->auxScore = s - 1;
      for (i=1; i<=maxAux; i++){
         p->auxLab[i] = NULL;
         p->auxScore[i] = 0.0;
      }  
   }
   return p;
}

/* EXPORT->AddLabel: append given label info to given label list */
LLink AddLabel(MemHeap *x, LabList *ll, LabId id,
               HTime st, HTime en, float score)
{
   LLink p,q,newLL;
   
   p = ll->tail->pred; q = ll->tail;
   newLL = CreateLabel(x,ll->maxAuxLab);
   newLL->labid = id; newLL->score = score; newLL->start = st; newLL->end = en;
   q->pred = newLL; newLL->succ = q;
   p->succ = newLL; newLL->pred = p;
   return newLL;
}

/* EXPORT->AddAuxLab: Store n auxiliary label/score in lab */
void AddAuxLab(LLink lab, int n, LabId *auxLab, float *auxScore)
{
   int i;
   
   for (i=1; i<=n; i++){
      lab->auxLab[i] = auxLab[i];
      lab->auxScore[i] = auxScore[i];
   }
}

/* EXPORT->DeleteLabel: unlink given label from a label list */
void DeleteLabel(LLink item)
{
   LLink p,q;
   
   if (item->pred == NULL || item->succ == NULL)
      HError(6571,"DeleteLabel: attempt to delete sentinel");
   p = item->pred; q = item->succ;
   q->pred = p; p->succ = q;
}


/* EXPORT->NumCases: find num cases of primary label in label list */
int NumCases(LabList *ll, LabId id)
{
   int n = 0;
   LLink p;
   
   for (p=ll->head->succ; p->succ!=NULL; p=p->succ) 
      if (p->labid == id) ++n;
   return n;
}

/* EXPORT->GetCase: find nth occ of primary label in label list */
LLink GetCase(LabList *ll, LabId id, int n)
{
   LLink p;
   int k=0;
   
   for (p=ll->head->succ; p->succ!=NULL; p=p->succ) {
      if (p->labid == id) 
         if (++k==n) return p;
   }
   HError(6571,"GetCase: %d case of %s nonexistent",n,id->name);
   return NULL;
}

/* EXPORT->NumAuxCases: find num cases of aux label i in label list */
int NumAuxCases(LabList *ll, LabId id, int i)
{
   int n = 0;
   LLink p;
   
   if (ll->maxAuxLab < i)
      HError(6570,"NumAuxCases: aux idx %d > max[%d]",i,ll->maxAuxLab);
   for (p=ll->head->succ; p->succ!=NULL; p=p->succ) 
      if ((i==0) ? (p->labid==id) : (p->auxLab[i]==id))  ++n;
   return n;
}

/* EXPORT->GetAuxCase: find nth occ of aux label i in label list */
LLink GetAuxCase(LabList *ll, LabId id, int n, int i)
{
   LLink p;
   int k=0;
   
   if (ll->maxAuxLab < i)
      HError(6570,"GetAuxCase: aux idx %d > max[%d]",i,ll->maxAuxLab);
   for (p=ll->head->succ; p->succ!=NULL; p=p->succ) {
      if ((i==0) ? (p->labid==id) : (p->auxLab[i]==id))
         if (++k==n) return p;
   }
   HError(6571,"GetAuxCase: %d case of %s nonexistent",n,id->name);
   return NULL;
}

/* EXPORT->GetLabN: return n'th primary label */
LLink GetLabN(LabList *ll, int n)
{
   int count=0;
   LLink p;

   for (p=ll->head->succ; p->succ!= NULL; p=p->succ) 
      if (++count==n) return p;
   HError(6571,"GetLabN: %d'th label nonexistent",n);
   return NULL;
}

/* EXPORT->GetAuxLabN: return n'th aux i label */
LLink GetAuxLabN(LabList *ll, int n, int i)
{
   int count=0;
   LLink p;

   if (ll->maxAuxLab < i)
      HError(6570,"GetAuxLabN: aux idx %d > max[%d]",i,ll->maxAuxLab);
   for (p=ll->head->succ; p->succ!= NULL; p=p->succ) 
      if (i==0 || p->auxLab[i]!=NULL)      
         if (++count==n) return p;
   HError(6571,"GetAuxLabN: %d'th aux[%d] label nonexistent",n,i);
   return NULL;
}

/* EXPORT->CountLabs: count num primary labels in lab list */
int CountLabs(LabList *ll)
{
   int count = 0;
   LLink p;

   if (ll!=NULL)
      for (p=ll->head->succ; p->succ!= NULL; p=p->succ) 
         ++count;
   return(count);
}

/* EXPORT->CountAuxLabs: count num aux i labels in lab list */
int CountAuxLabs(LabList *ll, int i)
{
   int count = 0;
   LLink p;

   if (ll!=NULL)
      for (p=ll->head->succ; p->succ != NULL; p=p->succ)
         if (i==0 || p->auxLab[i]!=NULL)
            ++count;
   return(count);
}

/* EXPORT->AuxLabEndTime: return the end time for the i'th aux lab */
HTime AuxLabEndTime(LLink p, int i)
{
   LLink q;
   
   q = p->succ;
   while (i!=0 && q->succ != NULL && q->auxLab[i]==NULL){
      p = q; q = q->succ;
   }
   return p->end;
}

/* PrintLabel: print the given label on one line */
static void PrintLabel(LLink p, int maxAux)
{
   int n;
   LabId id;
   
   printf("%8.0f%8.0f",p->start,p->end);
   printf(" %8s %5f",p->labid->name,p->score);
   for (n=1; n<=maxAux; n++){
      id = p->auxLab[n];
      printf(" %8s %5f",(id!=NULL)?id->name:"<null>",p->auxScore[n]);
   }