htklattice.cc

这是一款很好用的工具包

Boolean
Lattice::writeHTK(File &file, HTKScoreMapping scoreMapping,
						    Boolean printPosteriors)
{
    if (debug(DebugPrintFunctionality)) {
	dout()  << "Lattice::writeHTK: writing ";
    }

    fprintf(file, "# Header (generated by SRILM)\n");
    fprintf(file, "VERSION=%s\n", HTKLattice_Version);
    fprintf(file, "UTTERANCE="); printQuoted(file, name, htkheader.useQuotes);
    fputc('\n', file);
    fprintf(file, "base=%g\n", htkheader.logbase);
    fprintf(file, "dir=%s\n", "f");		// forward lattice

    double logscale = 1.0 / ProbToLogP(htkheader.logbase);

    /* 
     * Ancillary header information preserved from readHTK()
     */
    if (htkheader.tscale != HTK_def_tscale) {
	fprintf(file, "tscale=%g\n", htkheader.tscale);
    }
    if (htkheader.acscale != HTK_def_acscale) {
	fprintf(file, "acscale=%g\n", htkheader.acscale);
    }
    if (htkheader.lmscale != HTK_def_lmscale) {
	fprintf(file, "lmscale=%g\n", htkheader.lmscale);
    }
    if (htkheader.ngscale != HTK_def_ngscale) {
	fprintf(file, "ngscale=%g\n", htkheader.ngscale);
    }
    if (htkheader.prscale != HTK_def_prscale) {
	fprintf(file, "prscale=%g\n", htkheader.prscale);
    }
    if (htkheader.wdpenalty != HTK_def_wdpenalty) {
	fprintf(file, "wdpenalty=%lg\n", scaleHTKScore(htkheader.wdpenalty, logscale));
    }
    if (htkheader.duscale != HTK_def_duscale) {
	fprintf(file, "duscale=%g\n", htkheader.duscale);
    }
    if (htkheader.x1scale != HTK_def_xscale) {
	fprintf(file, "x1scale=%g\n", htkheader.x1scale);
    }
    if (htkheader.x2scale != HTK_def_xscale) {
	fprintf(file, "x2scale=%g\n", htkheader.x2scale);
    }
    if (htkheader.x3scale != HTK_def_xscale) {
	fprintf(file, "x3scale=%g\n", htkheader.x3scale);
    }
    if (htkheader.x4scale != HTK_def_xscale) {
	fprintf(file, "x4scale=%g\n", htkheader.x4scale);
    }
    if (htkheader.x5scale != HTK_def_xscale) {
	fprintf(file, "x5scale=%g\n", htkheader.x5scale);
    }
    if (htkheader.x6scale != HTK_def_xscale) {
	fprintf(file, "x6scale=%g\n", htkheader.x6scale);
    }
    if (htkheader.x7scale != HTK_def_xscale) {
	fprintf(file, "x7scale=%g\n", htkheader.x7scale);
    }
    if (htkheader.x8scale != HTK_def_xscale) {
	fprintf(file, "x8scale=%g\n", htkheader.x8scale);
    }
    if (htkheader.x9scale != HTK_def_xscale) {
	fprintf(file, "x9scale=%g\n", htkheader.x9scale);
    }
    if (htkheader.amscale != HTK_undef_float && printPosteriors) {
	fprintf(file, "amscale=%g\n", htkheader.amscale);
    }
    if (htkheader.hmms != 0) {
	fprintf(file, "hmms=");
	printQuoted(file, htkheader.hmms, htkheader.useQuotes);
	fputc('\n', file);
    }
    if (htkheader.lmname != 0) {
	fprintf(file, "lmname=");
	printQuoted(file, htkheader.lmname, htkheader.useQuotes);
	fputc('\n', file);
    }
    if (htkheader.ngname != 0) {
	fprintf(file, "ngname=");
	printQuoted(file, htkheader.ngname, htkheader.useQuotes);
	fputc('\n', file);
    }
    if (htkheader.vocab != 0) {
	fprintf(file, "vocab=");
	printQuoted(file, htkheader.vocab, htkheader.useQuotes);
	fputc('\n', file);
    }
	
    /*
     * We remap the internal node indices to consecutive unsigned integers
     * to allow a compact output representation.
     * We iterate over all nodes, renumbering them, and also counting the
     * number of transitions overall.
     * (Nodes which can be treated as transitions are not added as nodes.)
     */
    LHash<NodeIndex,unsigned> nodeMap;		// map nodeIndex to unsigned
    LHash<NodeIndex,Boolean> treatNodeAsTrans;	// keep a hash of nodes to be 
                                                //  printed only as transitions
    unsigned numNodes = 0;
    unsigned numTransitions = 0;

    LHashIter<NodeIndex, LatticeNode> nodeIter(nodes, nodeSort);
    NodeIndex nodeIndex;

    if (!htkheader.wordsOnNodes && !htkheader.scoresOnNodes) {
	// store nodes that can be treated as transitions for future reference
	while (LatticeNode *node = nodeIter.next(nodeIndex)) {
	    if (allowAsTrans(*this, nodeIndex)) {
		*treatNodeAsTrans.insert(nodeIndex) = true;
	    }
	}
    }

    // count number of nodes and transitions
    nodeIter.init();
    while (LatticeNode *node = nodeIter.next(nodeIndex)) {
	if(treatNodeAsTrans.find(nodeIndex)) {
	    numTransitions ++;
	} else {
	    *nodeMap.insert(nodeIndex) = numNodes ++;
	
	    NodeIndex toNodeIndex;
	    TRANSITER_T<NodeIndex,LatticeTransition>
		transIter(node->outTransitions);
	    while (LatticeTransition *trans = transIter.next(toNodeIndex)) {
		// only count transitions here when the destination node
		// is not being treated as a transition
		if(!treatNodeAsTrans.find(toNodeIndex)) {
		    numTransitions ++;
		}
	    }
	}
    }

    if (initial != NoNode) {
	fprintf(file, "start=%u\n",  *nodeMap.find(initial));
    }
    if (final != NoNode) {
	fprintf(file, "end=%u\n",  *nodeMap.find(final));
    }
    fprintf(file, "NODES=%u LINKS=%u\n", numNodes, numTransitions);

    if (debug(DebugPrintFunctionality)) {
      dout()  << numNodes << " nodes, "
	      << numTransitions << " transitions\n";
    }

    fprintf(file, "# Nodes\n");

    nodeIter.init(); 
    while (LatticeNode *node = nodeIter.next(nodeIndex)) {
        // skip printing this node if it can be treated as just a transition
        if (treatNodeAsTrans.find(nodeIndex)) continue;

	fprintf(file, "I=%u", *nodeMap.find(nodeIndex));

 	if (htkheader.wordsOnNodes) {
	    fprintf(file, "\tW=");
	    printQuoted(file, (node->word == vocab.ssIndex() ||
			       node->word == vocab.seIndex() ||
			       node->word == Vocab_None) ?
				    HTK_null_word : vocab.getWord(node->word),
			htkheader.useQuotes);
	}

	if (node->htkinfo != 0) {
	    HTKWordInfo &htkinfo = *node->htkinfo;

	    if (htkinfo.time != HTK_undef_float) {
		fprintf(file, "\tt=%g", htkinfo.time);
	    }
	    if (htkheader.scoresOnNodes) {
		writeScoreInfo(file, htkinfo, scoreMapping, logscale);
	    }
	    if (htkheader.wordsOnNodes) {
		writeWordInfo(file, htkinfo);
	    }
	}
	if (printPosteriors) {
	    fprintf(file, "\tp=%lg", (double)LogPtoProb(node->posterior));
	}
	fprintf(file, "\n");
    }

    fprintf(file, "# Links\n");

    unsigned linkNumber = 0;
    nodeIter.init(); 
    while (LatticeNode *node = nodeIter.next(nodeIndex)) {
      // if this node can be treated as a transition, print it as one and 
      // don't print it's own transitions as HTK transitions
      if (treatNodeAsTrans.find(nodeIndex)) {
	
	// get this node's neighboring nodes
	TRANSITER_T<NodeIndex,LatticeTransition> 
	    outTransIter(node->outTransitions);
	NodeIndex nextIndex;
	outTransIter.next(nextIndex);
	LatticeNode *nextNode = findNode(nextIndex); 
	assert(nextNode != 0);
	
	TRANSITER_T<NodeIndex,LatticeTransition> 
	    inTransIter(node->inTransitions);
	NodeIndex prevIndex;
	inTransIter.next(prevIndex);
	LatticeNode *prevNode = findNode(prevIndex); 
	assert(prevNode != 0);
	
	unsigned *toNodeId = nodeMap.find(nextIndex); 
	assert(toNodeId != 0);

	unsigned *fromNodeId = nodeMap.find(prevIndex); 
	assert(fromNodeId != 0);
	
	fprintf(file, "J=%u\tS=%u\tE=%u", linkNumber++, *fromNodeId, *toNodeId);
	
	if (!htkheader.wordsOnNodes) {
	    fprintf(file, "\tW=");
	    printQuoted(file, (node->word == vocab.ssIndex() ||
			       node->word == vocab.seIndex() ||
			       node->word == Vocab_None) ?
			HTK_null_word : vocab.getWord(node->word),
			htkheader.useQuotes);
	}
	
	if (node->htkinfo != 0) {
	    HTKWordInfo &htkinfo = *node->htkinfo;

	    writeScoreInfo(file, htkinfo, scoreMapping, logscale);
	    writeWordInfo(file, htkinfo);

	    if (scoreMapping != mapHTKngram &&
		htkinfo.ngram != HTK_undef_float)
	    {
		fprintf(file, "\tn=%lg", scaleHTKScore(htkinfo.ngram, logscale));
	    }
	    if (scoreMapping != mapHTKlanguage &&
		htkinfo.language != HTK_undef_float)
	    {
		fprintf(file, "\tl=%lg", scaleHTKScore(htkinfo.language, logscale));
	    }
	}
	
	/*
	 * map transition weight to one of the standard HTK scores
	 */
	if (scoreMapping != mapHTKnone) {
	    LatticeTransition *thisTrans =
				node->outTransitions.find(nextIndex);
	    assert(thisTrans != 0);
	    LatticeTransition *prevTrans =
				prevNode->outTransitions.find(nodeIndex);
	    assert(prevTrans != 0);

	    LogP combinedWeight = thisTrans->weight + 
				  prevTrans->weight;

	    fprintf(file, "\t%c=%lg",
		    (scoreMapping == mapHTKacoustic ? 'a' :
		     (scoreMapping == mapHTKngram ? 'n' :
		      (scoreMapping == mapHTKlanguage ? 'l' : '?'))),
		    scaleHTKScore(combinedWeight, logscale));
	}

	if (printPosteriors) {
	    fprintf(file, "\tp=%lg", (double)LogPtoProb(node->posterior));
	}
	
	fprintf(file, "\n");
      } else {
        // treat this node in the normal sense if it can't be treated solely
	// as a trans (but we have to ignore transitions to nodes that were
	// printed only as transitions)
	unsigned *fromNodeId = nodeMap.find(nodeIndex);

 	NodeIndex toNodeIndex;

	TRANSITER_T<NodeIndex,LatticeTransition>
					transIter(node->outTransitions);
	while (LatticeTransition *trans = transIter.next(toNodeIndex)) {
	    // skip printing this transition if the destination node is being
	    // treated as a transition (the transition weight is taken care of
	    // in printing of transition node case)
	    if (treatNodeAsTrans.find(toNodeIndex)) continue;

	    LatticeNode *toNode = findNode(toNodeIndex);
	    assert(toNode != 0);

	    unsigned *toNodeId = nodeMap.find(toNodeIndex); 
	    assert(toNodeId != 0);

	    fprintf(file, "J=%u\tS=%u\tE=%u",
				linkNumber++, *fromNodeId, *toNodeId);

	    if (!htkheader.wordsOnNodes) {
		fprintf(file, "\tW=");
		printQuoted(file, (toNode->word == vocab.ssIndex() ||
				   toNode->word == vocab.seIndex() ||
				   toNode->word == Vocab_None) ?
				   HTK_null_word : vocab.getWord(toNode->word),
			    htkheader.useQuotes);
	    }

	    if (toNode->htkinfo != 0) {
		HTKWordInfo &htkinfo = *toNode->htkinfo;

		if (!htkheader.scoresOnNodes) {
		    writeScoreInfo(file, htkinfo, scoreMapping, logscale);
		}
		if (!htkheader.wordsOnNodes){
		    writeWordInfo(file, htkinfo);
		}

		if (scoreMapping != mapHTKngram &&
		    htkinfo.ngram != HTK_undef_float)
		{
		    fprintf(file, "\tn=%lg", scaleHTKScore(htkinfo.ngram, logscale));
		}
		if (scoreMapping != mapHTKlanguage &&
		    htkinfo.language != HTK_undef_float)
		{
		    fprintf(file, "\tl=%lg", scaleHTKScore(htkinfo.language, logscale));
		}
	    }

	    /*
	     * map transition weight to one of the standard HTK scores
	     */
	    if (scoreMapping != mapHTKnone) {
		fprintf(file, "\t%c=%lg",
			    (scoreMapping == mapHTKacoustic ? 'a' :
			     (scoreMapping == mapHTKngram ? 'n' :
			      (scoreMapping == mapHTKlanguage ? 'l' : '?'))),
			    scaleHTKScore(trans->weight, logscale));
	    }

	    fprintf(file, "\n");
	  }
      }
    }

    return true;
}


/* 
 * Compute pronunciation scores
 * 	(for nodes with HTKWordInfo that have phone backtraces)
 */
Boolean
Lattice::scorePronunciations(VocabMultiMap &dictionary, Boolean intlogs)
{
    if (debug(DebugPrintFunctionality)) {
      dout() << "Lattice::scorePronunciations: starting\n";
    }

    Vocab &phoneVocab = dictionary.vocab2;

    /*
     * Go through all HTKWordInfo structures, extract the phone sequences,
     * and look up their probabilities in the dictionary
     */
    for (unsigned i = 0; i < htkinfos.size(); i ++) {
	HTKWordInfo *info = htkinfos[i];

	/*
	 * only rescore words that have pronunciations and a non-NULL
	 */
	if (info->div != 0) {
	    if (info->word == Vocab_None) {
		dout() << "Lattice::scorePronunciations: warning: " << name
		       << " has pronunciation on " << HTK_null_word << " node"
		       << " (time = " << info->time << ")\n";

		info->pron = LogP_Zero;
	    } else {
		/*
		 * parse the phone sequence from the string
		 * example:
		 * d=:#[s]t,0.12:s[t]r,0.03:t[r]ay,0.05:r[ay]k,0.09:ay[k]#,0.09:
		 * and convert into an index string
		 */
		makeArray(char, phoneString, strlen(info->div) + 1);
		strcpy(phoneString, info->div);

		Array<VocabIndex> phones;
		unsigned numPhones = 0;

		for (char *s = strtok(phoneString, phoneSeparator);
		     s != 0;
		     s = strtok(NULL, phoneSeparator))
		{
		    // skip empty components (at beginning and end)
		    if (s[0] == '\0') continue;

		    // strip duration part
		    char *e = strchr(s, ',');
		    if (e != 0) *e = '\0';

		    // strip context from triphone labels
		    e = strchr(s, '[');
		    if (e != 0) s = e + 1;

		    e = strrchr(s, ']');
		    if (e != 0) *e = '\0';

		    phones[numPhones ++] = phoneVocab.addWord(s);
		}
		phones[numPhones] = Vocab_None;

		// find pronunciation prob
		Prob p = dictionary.get(info->word, phones.data());

		if (p == 0.0) {
		    // missing pronunciation get score 0
		    info->pron = LogP_One;
		} else {
		    if (intlogs) {
			info->pron = IntlogToLogP(p);
		    } else {
			info->pron = ProbToLogP(p);
		    }
		}
	    }
	}
    }

    return true;
}