latticenbest.cc

这是一款很好用的工具包

	}
        hyps.push(expandedHyp);
      }
      
      if (succIndex < nodeInfo->numSuccs - 1) {
        double score = topHyp->forwardProb +
				nodeInfo->succs[succIndex + 1].bwScore;
        
        LatticeNBestHyp *expandedHyp =
            new LatticeNBestHyp(score, topHyp->forwardProb, topHyp->nodeIndex,
	    			succIndex, false, topHyp->nbestPath,
				topHyp->wordCnt, topHyp->acoustic,
				topHyp->ngram, topHyp->language,
                                topHyp->pron, topHyp->duration,
				topHyp->xscore1, topHyp->xscore2,
				topHyp->xscore3, topHyp->xscore4,
				topHyp->xscore5, topHyp->xscore6,
				topHyp->xscore7, topHyp->xscore8,
				topHyp->xscore9);
        
        assert(expandedHyp != 0);
	if (maxHyps > 0 && hyps.size() >= maxHyps) {
	  LatticeNBestHyp *pruneHyp = hyps.top_min(); // get hyp
	  hyps.pop_min();
	  delete pruneHyp;
	  if (debug(DebugPrintOutLoop) ||
	      firstPruned && debug(DebugPrintFunctionality))
	  {
	    dout() << "Lattice::computeNBest: max number of hyps reached, pruning lowest score hyp\n";
	    firstPruned = false;
	  }
	}
	hyps.push(expandedHyp);
      }
    }
    delete topHyp;
  }

  // pop remaining hyps to free memory
  while (!hyps.empty()) {
    LatticeNBestHyp *topHyp = hyps.top_max();
    hyps.pop_max();
    delete topHyp;
  }
  delete [] viterbiForwardProbs;
  delete [] viterbiBackwardProbs;
  delete [] nodeInfos;

  return true;
}


/* *************************
 * Viterbi N-best generation
 * ************************* */

typedef LHash<const char *, LatticeNBestHyp *> ENTRYHASH_T;
typedef LHashIter<const char *, LatticeNBestHyp *> ENTRYITER_T;

#ifdef INSTANTIATE_TEMPLATES
INSTANTIATE_LHASH(const char *, LatticeNBestHyp*);
#endif

struct HypEntry { const char * key; LatticeNBestHyp * hyp; };
static int compEntry(const void *p1, const void *p2) 
{
    LogP pr1 = ((const HypEntry *) p1)->hyp->forwardProb;
    LogP pr2 = ((const HypEntry *) p2)->hyp->forwardProb;

    if (pr1 == pr2) return 0;
    else if (pr1 < pr2) return 1;
    else return -1;
}

Boolean
Lattice::computeNBestViterbi(unsigned N, NBestOptions &nbestOut,
						SubVocab &ignoreWords,
			     			const char *multiwordSeparator)
{
    /*
     * topological sort
     */
    unsigned numNodes = getNumNodes(); 
    
    NodeIndex *sortedNodes = new NodeIndex[numNodes];
    assert(sortedNodes != 0);
    unsigned numReachable = sortNodes(sortedNodes);
    
    if (numReachable != numNodes) {
        dout() << "Lattice::computeNBestViterbi: warning: called with unreachable nodes\n";
    }
    
    if (sortedNodes[0] != initial) {
        dout() << "Lattice::computeNBestViterbi: initial node is not first\n";
        delete [] sortedNodes;
        return LogP_Inf;
    }
    
    unsigned finalPosition = 0;
    for (finalPosition = 1; finalPosition < numReachable; finalPosition ++) {
        if (sortedNodes[finalPosition] == final) break;
    }

    if (finalPosition == numReachable) {
        dout() << "Lattice::computeNBestViterbi: final node is not reachable\n";
        delete [] sortedNodes;
        return LogP_Inf;
    }

    ENTRYHASH_T ht(2 * N);

    int i, j;
    Array<HypEntry> **hyps = new Array<HypEntry> * [numReachable];
    assert(hyps != 0);
    memset(hyps, 0, sizeof(void *) * numReachable);

    int *map = new int [maxIndex];
    assert(map != 0);
    memset(map, 0, sizeof(int) * maxIndex);

    for (i = 0; i <= finalPosition; i++) {
        map[sortedNodes[i]] = i;
    }

    Array<int> *freeList = new Array<int> [numReachable];
    assert(freeList != 0);

    for (i = 0; i < finalPosition; i++) {
        NodeIndex nodeIndex = sortedNodes[i];
        LatticeNode *node = nodes.find(nodeIndex);
        assert(node != 0);

        TRANSITER_T<NodeIndex, LatticeTransition> transIter(node->outTransitions);
        NodeIndex toNodeIndex;
        NodeIndex maxTo = 0;
        while (transIter.next(toNodeIndex)) {
            if (map[toNodeIndex] > maxTo) {
                maxTo = map[toNodeIndex];
            }
        }
        
        Array<int> &fl = freeList[maxTo];
        fl[fl.size()] = i;
    }

    /* initialize initial node */
    LatticeNBestPath *initialPath = new LatticeNBestPath(initial, 0);
    assert(initialPath != 0);

    LatticeNBestHyp *initialHyp =
        new LatticeNBestHyp(LogP_One, LogP_One, initial, -1,
                            false, initialPath, 0, 
                            LogP_One, LogP_One, LogP_One, LogP_One,
			    LogP_One, LogP_One, LogP_One, LogP_One,
			    LogP_One, LogP_One, LogP_One, LogP_One,
			    LogP_One, LogP_One);

    
    char *initialKey = (char *)"";
    hyps[0] = new Array<HypEntry> (0, 1);
    assert(hyps[0] != 0);
    HypEntry *e = hyps[0]->data();
    e->key = strdup(initialKey);
    e->hyp = initialHyp;
    
    for (i = 1; i <= finalPosition; i++) {
        NodeIndex nodeIndex = sortedNodes[i];
        LatticeNode *node = nodes.find(nodeIndex);
        assert(node != 0);

        Boolean isFinal = (nodeIndex == final);

        const char *word = "";
        if (!ignoreWord(node->word) &&
            !ignoreWords.getWord(node->word) &&
            !vocab.isNonEvent(node->word) &&
            node->word != vocab.seIndex()) 
          word = getWord(node->word);
        
        int len = strlen(word);

        unsigned wcnt = (len > 0 ? 1 : 0); 	// word count (ignore non-words)
	LogP acoustic = LogP_One;	// acoustic model log score
	LogP ngram    = LogP_One;	// ngram model log score
	LogP language = LogP_One;	// language model log score
	LogP pron     = LogP_One;	// pronunciation log score
	LogP duration = LogP_One;	// duration log score
	LogP xscore1  = LogP_One; 	// extra score #1
	LogP xscore2  = LogP_One; 	// extra score #2
	LogP xscore3  = LogP_One; 	// extra score #3
	LogP xscore4  = LogP_One; 	// extra score #4
	LogP xscore5  = LogP_One; 	// extra score #5
	LogP xscore6  = LogP_One; 	// extra score #6
	LogP xscore7  = LogP_One; 	// extra score #7
	LogP xscore8  = LogP_One; 	// extra score #8
	LogP xscore9  = LogP_One; 	// extra score #9

	if (node->htkinfo) {
            if (node->htkinfo->acoustic != HTK_undef_float) 
              acoustic = node->htkinfo->acoustic;
            if (node->htkinfo->ngram != HTK_undef_float) 
              ngram    = node->htkinfo->ngram;
            if (node->htkinfo->language != HTK_undef_float) 
              language = node->htkinfo->language;
            if (node->htkinfo->pron != HTK_undef_float)
              pron     = node->htkinfo->pron;
            if (node->htkinfo->duration != HTK_undef_float) 
              duration = node->htkinfo->duration;
            if (node->htkinfo->xscore1 != HTK_undef_float) 
              xscore1  = node->htkinfo->xscore1;
            if (node->htkinfo->xscore2 != HTK_undef_float) 
              xscore2  = node->htkinfo->xscore2;
            if (node->htkinfo->xscore3 != HTK_undef_float) 
              xscore3  = node->htkinfo->xscore3;
            if (node->htkinfo->xscore4 != HTK_undef_float) 
              xscore4  = node->htkinfo->xscore4;
            if (node->htkinfo->xscore5 != HTK_undef_float) 
              xscore5  = node->htkinfo->xscore5;
            if (node->htkinfo->xscore6 != HTK_undef_float) 
              xscore6  = node->htkinfo->xscore6;
            if (node->htkinfo->xscore7 != HTK_undef_float) 
              xscore7  = node->htkinfo->xscore7;
            if (node->htkinfo->xscore8 != HTK_undef_float) 
              xscore8  = node->htkinfo->xscore8;
            if (node->htkinfo->xscore9 != HTK_undef_float) 
              xscore9  = node->htkinfo->xscore9;
	}

        int num;

        // propogate to successors
        TRANSITER_T<NodeIndex, LatticeTransition>
					transIter(node->inTransitions);
        transIter.init();
        LatticeTransition *inTrans;
        NodeIndex fromNodeIndex;
        while (inTrans = transIter.next(fromNodeIndex)) {

            int from = map[fromNodeIndex];
            
            Array<HypEntry> *fromArray = hyps[from];
            if (fromArray == 0) continue;
            num = fromArray->size();
            HypEntry *entries = fromArray->data();
                
            for (j = 0; j < num; j++) {
                LatticeNBestHyp *hyp = entries[j].hyp;
                LogP forwardProb = hyp->forwardProb + inTrans->weight;

                char *newkey = new char [strlen(entries[j].key) + len + 2];
		assert(newkey != 0);
            
                if (len) {
                  sprintf(newkey, "%s%c%s", entries[j].key, 
				multiwordSeparator ? *multiwordSeparator : ' ',
				word);
                } else {
                  strcpy(newkey, entries[j].key);
		}
                
                Boolean foundP;
                LatticeNBestHyp **p = ht.insert(newkey, foundP);

                if (foundP) {
                    // already exists
                    LatticeNBestHyp *oldHyp = *p;

                    if (oldHyp->score < forwardProb) {
                        oldHyp->score = forwardProb;
                        oldHyp->wordCnt  = wcnt + hyp->wordCnt;

                        oldHyp->nbestPath->pred->release();
                        oldHyp->nbestPath->pred = hyp->nbestPath;
                        hyp->nbestPath->linkto();
                    
                        oldHyp->forwardProb = forwardProb;
                        oldHyp->acoustic    = acoustic + hyp->acoustic;
                        oldHyp->ngram       = ngram + hyp->ngram;
                        oldHyp->language    = language + hyp->language;
                        oldHyp->pron        = pron + hyp->pron;
                        oldHyp->duration    = duration + hyp->duration;
                        oldHyp->xscore1     = xscore1 + hyp->xscore1;
                        oldHyp->xscore2     = xscore2 + hyp->xscore2;
                        oldHyp->xscore3     = xscore3 + hyp->xscore3;
                        oldHyp->xscore4     = xscore4 + hyp->xscore4;
                        oldHyp->xscore5     = xscore5 + hyp->xscore5;
                        oldHyp->xscore6     = xscore6 + hyp->xscore6;
                        oldHyp->xscore7     = xscore7 + hyp->xscore7;
                        oldHyp->xscore8     = xscore8 + hyp->xscore8;
                        oldHyp->xscore9     = xscore9 + hyp->xscore9;
                    }

                } else {
                    // new one
                    LatticeNBestPath *path =
				new LatticeNBestPath(nodeIndex, hyp->nbestPath);

                    LatticeNBestHyp *newHyp =
				new LatticeNBestHyp(forwardProb, forwardProb,
						     nodeIndex, -1, isFinal,
						     path, wcnt + hyp->wordCnt,
                                                     acoustic + hyp->acoustic,
						     ngram + hyp->ngram,
						     language + hyp->language,
						     pron + hyp->pron,
						     duration + hyp->duration,
						     xscore1 + hyp->xscore1,
						     xscore2 + hyp->xscore2,
						     xscore3 + hyp->xscore3,
						     xscore4 + hyp->xscore4,
						     xscore5 + hyp->xscore5,
						     xscore6 + hyp->xscore6,
						     xscore7 + hyp->xscore7,
						     xscore8 + hyp->xscore8,
						     xscore9 + hyp->xscore9);

                    *p = newHyp;
                }

                delete [] newkey;
            }
        }

        // copy entries to array
        num = ht.numEntries();

        if (num > 0) { 
            HypEntry *entries = new HypEntry [ num ];
	    assert(entries != 0);
            num = 0;

            ENTRYITER_T iter(ht);
            LatticeNBestHyp **phyp;
            const char *key;
            while (phyp = iter.next(key)) {
                entries[num].key = key;
                entries[num].hyp = *phyp;
                num++;
            }
            
            qsort(entries, num, sizeof(HypEntry), compEntry);
            if (num > N) num = N;
            
            hyps[i] = new Array<HypEntry> (0, num);
	    assert(hyps[i] != 0);
            HypEntry *dst = hyps[i]->data();
            HypEntry *src = entries;
            
            for (j = 0; j < num; j++, src++, dst++) {
                dst->key = strdup(src->key);
                dst->hyp = src->hyp;
            }
            
            for (j = num; j < ht.numEntries(); j++) {
                delete entries[j].hyp;
            }
            
            ht.clear();
            
            delete [] entries;
        }

        // free hyps in freeList
        Array<int> &fl = freeList[i];
        
        for (j = 0; j < fl.size(); j++) {
            int index = fl[j];
            
            if (hyps[index] == 0) continue;
            HypEntry *entries = hyps[index]->data();
            
            for (int k = hyps[index]->size()-1; k >= 0; k--) {
                delete entries[k].hyp;
                free ((char *) entries[k].key);
            }
            
            delete hyps[index];
            hyps[index] = 0;
        }
    }

    // output reasults
    if (hyps[finalPosition] && hyps[finalPosition]->size()) {
        Array<HypEntry> *results = hyps[finalPosition];
        
        int num = results->size();

        HypEntry *entries = results->data();

        qsort(entries, num, sizeof(HypEntry), compEntry);
        
        int i;
        for (i = 0; i < num && i < N; i++) {
            
            LatticeNBestHyp *hyp = entries[i].hyp;
            hyp->writeHyp(i, *this, nbestOut);

            delete hyp;
            free((void *)entries[i].key);
        }

        delete results;
        hyps[finalPosition] = 0;
    } else {        
        dout() << "Lattice::computeNBestViterbi: "
	       << "no hyp reached final!" << endl;
    }

    // free hyps and keys -- should not be necessary, but just in case
    for (i = 0; i <= finalPosition; i++) {
        if (hyps[i] == 0) continue;

        int num = hyps[i]->size();
        HypEntry *entries = hyps[i]->data();

        for (j = 0; j < num; j++) {
            delete entries[j].hyp;
            free((void *)entries[j].key);
        }

        delete hyps[i];
        hyps[i] = 0;
    }

    delete [] freeList;
    delete [] map;
    delete [] sortedNodes;
    delete [] hyps;

    return true;
}

NBestOptions::NBestOptions(char *myNbestOutDir,
			   char *myNbestOutDirNgram,
			   char *myNbestOutDirPron,
			   char *myNbestOutDirDur,
			   char *myNbestOutDirXscore1,
			   char *myNbestOutDirXscore2,
			   char *myNbestOutDirXscore3,
			   char *myNbestOutDirXscore4,
			   char *myNbestOutDirXscore5,
			   char *myNbestOutDirXscore6,
			   char *myNbestOutDirXscore7,
			   char *myNbestOutDirXscore8,
			   char *myNbestOutDirXscore9,
			   char *myNbestOutDirRttm)
  : nbestOutDir(myNbestOutDir), nbestOutDirNgram(myNbestOutDirNgram),
    nbestOutDirPron(myNbestOutDirPron), nbestOutDirDur(myNbestOutDirDur),
    nbestOutDirXscore1(myNbestOutDirXscore1),
    nbestOutDirXscore2(myNbestOutDirXscore2),
    nbestOutDirXscore3(myNbestOutDirXscore3),
    nbestOutDirXscore4(myNbestOutDirXscore4),
    nbestOutDirXscore5(myNbestOutDirXscore5),
    nbestOutDirXscore6(myNbestOutDirXscore6),
    nbestOutDirXscore7(myNbestOutDirXscore7),