latticenbest.cc

这是一款很好用的工具包

    nbestOutDirXscore8(myNbestOutDirXscore8),
    nbestOutDirXscore9(myNbestOutDirXscore9),
    nbestOutDirRttm(myNbestOutDirRttm),
    writingFiles(false), nbest(0), nbestNgram(0), nbestPron(0), nbestDur(0),
    nbestXscore1(0), nbestXscore2(0), nbestXscore3(0),
    nbestXscore4(0), nbestXscore5(0), nbestXscore6(0),
    nbestXscore7(0), nbestXscore8(0), nbestXscore9(0),
    nbestRttm(0)
{
}

NBestOptions::~NBestOptions()
{
    if (writingFiles) {
	closeFiles();
    }
}

static void
makeDir(const char *dir, Boolean overwrite)
{
    if (MKDIR(dir) < 0) {
      if (errno == EEXIST) {
	if (!overwrite) {
	  cerr << "Dir " << dir << " already exists, please give another one\n";
	  exit(2);
	}
      } else {
	perror(dir);
	exit(1);
      }
    }
}

Boolean
NBestOptions::makeDirs(Boolean overwrite)
{
    if (nbestOutDir) {
	makeDir(nbestOutDir, overwrite);
	if (nbestOutDirNgram) makeDir(nbestOutDirNgram, overwrite);
	if (nbestOutDirPron) makeDir(nbestOutDirPron, overwrite);
	if (nbestOutDirDur) makeDir(nbestOutDirDur, overwrite);
	if (nbestOutDirXscore1) makeDir(nbestOutDirXscore1, overwrite);
	if (nbestOutDirXscore2) makeDir(nbestOutDirXscore2, overwrite);
	if (nbestOutDirXscore3) makeDir(nbestOutDirXscore3, overwrite);
	if (nbestOutDirXscore4) makeDir(nbestOutDirXscore4, overwrite);
	if (nbestOutDirXscore5) makeDir(nbestOutDirXscore5, overwrite);
	if (nbestOutDirXscore6) makeDir(nbestOutDirXscore6, overwrite);
	if (nbestOutDirXscore7) makeDir(nbestOutDirXscore7, overwrite);
	if (nbestOutDirXscore8) makeDir(nbestOutDirXscore8, overwrite);
	if (nbestOutDirXscore9) makeDir(nbestOutDirXscore9, overwrite);
	if (nbestOutDirRttm) makeDir(nbestOutDirRttm, overwrite);
	return true;
    } else {
	cerr << "Warning: no nbest output directory specified\n";
	return false;
    }
}

Boolean
NBestOptions::openFiles(const char *name)
{
  if (!writingFiles) {
    // make sure all the File pointers are empty
    assert(nbest == 0 &&
	   nbestNgram   == 0 &&
	   nbestPron    == 0 &&
	   nbestDur     == 0 &&
	   nbestXscore1 == 0 &&
	   nbestXscore2 == 0 &&
	   nbestXscore3 == 0 &&
	   nbestXscore4 == 0 &&
	   nbestXscore5 == 0 &&
	   nbestXscore6 == 0 &&
	   nbestXscore7 == 0 &&
	   nbestXscore8 == 0 &&
	   nbestXscore9 == 0 &&
	   nbestRttm    == 0);

    writingFiles = true;

    unsigned basenameLen = 1 + strlen(name) + sizeof(GZIP_SUFFIX);
    
    if (nbestOutDir) {
      makeArray(char, outfile, strlen(nbestOutDir) + basenameLen);
      sprintf(outfile, "%s/%s%s", nbestOutDir, name, GZIP_SUFFIX);
      nbest = new File(outfile, "w");
    }
    if (nbestOutDirNgram) {
      makeArray(char, outfile, strlen(nbestOutDirNgram) + basenameLen);
      sprintf(outfile, "%s/%s%s", nbestOutDirNgram, name, GZIP_SUFFIX);
      nbestNgram = new File(outfile, "w");
    }
    if (nbestOutDirPron) {
      makeArray(char, outfile, strlen(nbestOutDirPron) + basenameLen);
      sprintf(outfile, "%s/%s%s", nbestOutDirPron, name, GZIP_SUFFIX);
      nbestPron = new File(outfile, "w");
    }
    if (nbestOutDirDur) {
      makeArray(char, outfile, strlen(nbestOutDirDur) + basenameLen);
      sprintf(outfile, "%s/%s%s", nbestOutDirDur, name, GZIP_SUFFIX);
      nbestDur = new File(outfile, "w");
    }
    if (nbestOutDirXscore1) {
      makeArray(char, outfile, strlen(nbestOutDirXscore1) + basenameLen);
      sprintf(outfile, "%s/%s%s", nbestOutDirXscore1, name, GZIP_SUFFIX);
      nbestXscore1 = new File(outfile, "w");
    }
    if (nbestOutDirXscore2) {
      makeArray(char, outfile, strlen(nbestOutDirXscore2) + basenameLen);
      sprintf(outfile, "%s/%s%s", nbestOutDirXscore2, name, GZIP_SUFFIX);
      nbestXscore2 = new File(outfile, "w");
    }
    if (nbestOutDirXscore3) {
      makeArray(char, outfile, strlen(nbestOutDirXscore3) + basenameLen);
      sprintf(outfile, "%s/%s%s", nbestOutDirXscore3, name, GZIP_SUFFIX);
      nbestXscore3 = new File(outfile, "w");
    }
    if (nbestOutDirXscore4) {
      makeArray(char, outfile, strlen(nbestOutDirXscore4) + basenameLen);
      sprintf(outfile, "%s/%s%s", nbestOutDirXscore4, name, GZIP_SUFFIX);
      nbestXscore4 = new File(outfile, "w");
    }
    if (nbestOutDirXscore5) {
      makeArray(char, outfile, strlen(nbestOutDirXscore5) + basenameLen);
      sprintf(outfile, "%s/%s%s", nbestOutDirXscore5, name, GZIP_SUFFIX);
      nbestXscore5 = new File(outfile, "w");
    }
    if (nbestOutDirXscore6) {
      makeArray(char, outfile, strlen(nbestOutDirXscore6) + basenameLen);
      sprintf(outfile, "%s/%s%s", nbestOutDirXscore6, name, GZIP_SUFFIX);
      nbestXscore6 = new File(outfile, "w");
    }
    if (nbestOutDirXscore7) {
      makeArray(char, outfile, strlen(nbestOutDirXscore7) + basenameLen);
      sprintf(outfile, "%s/%s%s", nbestOutDirXscore7, name, GZIP_SUFFIX);
      nbestXscore7 = new File(outfile, "w");
    }
    if (nbestOutDirXscore8) {
      makeArray(char, outfile, strlen(nbestOutDirXscore8) + basenameLen);
      sprintf(outfile, "%s/%s%s", nbestOutDirXscore8, name, GZIP_SUFFIX);
      nbestXscore8 = new File(outfile, "w");
    }
    if (nbestOutDirXscore9) {
      makeArray(char, outfile, strlen(nbestOutDirXscore9) + basenameLen);
      sprintf(outfile, "%s/%s%s", nbestOutDirXscore9, name, GZIP_SUFFIX);
      nbestXscore9 = new File(outfile, "w");
    }
    if (nbestOutDirRttm) {
      makeArray(char, outfile, strlen(nbestOutDirRttm) + basenameLen);
      sprintf(outfile, "%s/%s%s", nbestOutDirRttm, name, GZIP_SUFFIX);
      nbestRttm = new File(outfile, "w");
    }
    return true;
  } else {
    cerr << "Already have file open for lattice " << name
         << ", not opening another set\n";
    return false;
  }
}

Boolean
NBestOptions::closeFiles()
{
  if (writingFiles) {
    delete nbest;
    delete nbestNgram;
    delete nbestPron;
    delete nbestDur;
    delete nbestXscore1;
    delete nbestXscore2;
    delete nbestXscore3;
    delete nbestXscore4;
    delete nbestXscore5;
    delete nbestXscore6;
    delete nbestXscore7;
    delete nbestXscore8;
    delete nbestXscore9;
    delete nbestRttm;

    nbest        = 0;
    nbestNgram   = 0;
    nbestPron    = 0;
    nbestDur     = 0;
    nbestXscore1 = 0;
    nbestXscore2 = 0;
    nbestXscore3 = 0;
    nbestXscore4 = 0;
    nbestXscore5 = 0;
    nbestXscore6 = 0;
    nbestXscore7 = 0;
    nbestXscore8 = 0;
    nbestXscore9 = 0;
    nbestRttm    = 0;

    writingFiles = false;
    return true;
  } else {
    cerr << "Warning: File close method called when files are not opened\n";
    return false;
  }
}

LatticeNBestPath::LatticeNBestPath(NodeIndex node, LatticeNBestPath *pred)
  : node(node), pred(pred), numReferences(0)
{
    if (pred != 0) {
    	pred->linkto();
    }
}

LatticeNBestPath::~LatticeNBestPath()
{
    if (pred != 0) {
	pred->release();
    }
}

void
LatticeNBestPath::linkto()
{
    numReferences += 1;
}

void
LatticeNBestPath::release()
{
    assert(numReferences > 0);
    numReferences -= 1;
    if (numReferences == 0) {
	delete this;
    }
}

unsigned
LatticeNBestPath::getPath(Array<NodeIndex> &result)
{
    unsigned startIndex = result.size();

    // extract path in reverse
    for (LatticeNBestPath *current = this;
         current != 0;
	 current = current->pred)
    {
    	result[result.size()] = current->node;
    }

    // reverse order
    for (unsigned i = startIndex, j = result.size() - 1; i < j; i ++, j --) {
    	NodeIndex h = result[i];
	result[i] = result[j];
	result[j] = h;
    }

    return result.size() - startIndex;
}

LatticeNBestHyp::LatticeNBestHyp(double myScore, LogP myForwardProb, 
                                 NodeIndex myNodeIndex, int mySuccIndex,
				 Boolean myEndOfSent,
				 LatticeNBestPath *myNBestPath,
				 unsigned myWordCnt,
				 LogP myAcoustic, LogP myNgram, LogP myLanguage,
				 LogP myPron, LogP myDuration, 
				 LogP myXscore1, LogP myXscore2, LogP myXscore3,
				 LogP myXscore4, LogP myXscore5, LogP myXscore6,
				 LogP myXscore7, LogP myXscore8, LogP myXscore9)
  :  score(myScore), forwardProb(myForwardProb), 
     nodeIndex(myNodeIndex), succIndex(mySuccIndex), endOfSent(myEndOfSent),
     nbestPath(myNBestPath), wordCnt(myWordCnt), acoustic(myAcoustic),
     ngram(myNgram), language(myLanguage), pron(myPron), duration(myDuration), 
     xscore1(myXscore1), xscore2(myXscore2), xscore3(myXscore3),
     xscore4(myXscore4), xscore5(myXscore5), xscore6(myXscore6),
     xscore7(myXscore7), xscore8(myXscore8), xscore9(myXscore9)
{
    if (nbestPath != 0) {
	nbestPath->linkto();
    }
}

LatticeNBestHyp::~LatticeNBestHyp()
{
    if (nbestPath != 0) {
        nbestPath->release();
    }
}

Boolean
LatticeNBestHyp::writeHyp(int hypNum, Lattice &lat, NBestOptions &nbestOut)
{

  VocabIndex HTK_SU = lat.vocab.getIndex("<su>");
  if (HTK_SU == Vocab_None) {
    // try capitalized
    HTK_SU =lat.vocab.getIndex("<SU>");
    if (HTK_SU == Vocab_None) {
      //cerr << "Could not find SU word, index will be Vocab_None\n";
    }
  }

  assert(nbestOut.writingFiles);
  if (nbestOut.nbestOutDir) {
    fprintf(*nbestOut.nbest, "%g %g %u ", acoustic, language, wordCnt);

    char *speaker, channel, *time, *session;
    float start_time;

    if (nbestOut.nbestRttm) {
      speaker = strdup(lat.getName());
      char *ptr  = strchr(speaker, '_');
      char *ptr2 = strchr(++ptr, '_');
      char *ptr3 = strchr(++ptr2, '_');
      
      assert(ptr3 != NULL);
      ptr3[0] = '\0'; // end string so that new string is 'corpus'_'ses'_'spkr'
      time = strdup(++ptr3);
      char *end  = strchr(time, '_');
      end[0] = '\0';
      start_time = atof(time) / 1000;
      float end_time  = atof(++end) / 1000;

      channel = ptr2[0];
      session = strdup(speaker);
      char *ptr4 = strrchr(session, '_');
      assert(ptr4 != NULL);
      ptr4[0] = '\0'; // end string so that new string is 'corpus'_'ses'
    }

    Array<NodeIndex> path;
    nbestPath->getPath(path);

    for (int n = 0; n < path.size(); n++) {
      LatticeNode *thisNode = lat.findNode(path[n]);
      LatticeNode *prevNode = lat.findNode(path[(n>0 ? n-1 : 0)]);
      assert(thisNode != 0 && prevNode != 0);

      if (thisNode->word != Vocab_None) {
	fprintf(*nbestOut.nbest, "%s ", lat.getWord(thisNode->word));
	
	//LEXEME sw_47411 2 51.670 0.470 Yeah lex       SW_47411_B <NA>
	//SU     sw_47411 2 52.140 1.930 <NA> statement SW_47411_B <NA>
	if (nbestOut.nbestRttm) {
	  if (thisNode->word == HTK_SU) {
	    if (thisNode->htkinfo) {
	      fprintf(*nbestOut.nbestRttm,
		    "%d SU     %s %c %.2f X <NA> statement %s <NA>\n",
		    hypNum, session, channel,
		    start_time+thisNode->htkinfo->time, speaker); 	
	    }
	  } else {
	    if (thisNode->htkinfo && prevNode->htkinfo) {
	      fprintf(*nbestOut.nbestRttm,
	            "%d LEXEME %s %c %.2f %.2f %s lex %s <NA>\n",
		    hypNum, session, channel,
		    start_time+prevNode->htkinfo->time,
		    thisNode->htkinfo->time-prevNode->htkinfo->time,
		    lat.getWord(thisNode->word), speaker);
	    }
	  }
	}
      }
    }
    fprintf(*nbestOut.nbest, "\n");

    if (nbestOut.nbestRttm) {
      free(speaker);    
      free(session);
      free(time);
    }

    if (nbestOut.nbestOutDirNgram) {
      fprintf(*nbestOut.nbestNgram, "%g\n", ngram);
    }
    if (nbestOut.nbestOutDirPron) {
      fprintf(*nbestOut.nbestPron, "%g\n", pron);
    }
    if (nbestOut.nbestOutDirDur) {
      fprintf(*nbestOut.nbestDur, "%g\n", duration);
    }
    if (nbestOut.nbestOutDirXscore1) {
      fprintf(*nbestOut.nbestXscore1, "%g\n", xscore1);
    }
    if (nbestOut.nbestOutDirXscore2) {
      fprintf(*nbestOut.nbestXscore2, "%g\n", xscore2);
    }
    if (nbestOut.nbestOutDirXscore3) {
      fprintf(*nbestOut.nbestXscore3, "%g\n", xscore3);
    }
    if (nbestOut.nbestOutDirXscore4) {
      fprintf(*nbestOut.nbestXscore4, "%g\n", xscore4);
    }
    if (nbestOut.nbestOutDirXscore5) {
      fprintf(*nbestOut.nbestXscore5, "%g\n", xscore5);
    }
    if (nbestOut.nbestOutDirXscore6) {
      fprintf(*nbestOut.nbestXscore6, "%g\n", xscore6);
    }
    if (nbestOut.nbestOutDirXscore7) {
      fprintf(*nbestOut.nbestXscore7, "%g\n", xscore7);
    }
    if (nbestOut.nbestOutDirXscore8) {
      fprintf(*nbestOut.nbestXscore8, "%g\n", xscore8);
    }
    if (nbestOut.nbestOutDirXscore9) {
      fprintf(*nbestOut.nbestXscore9, "%g\n", xscore9);
    }

    return true;
  } else {
    cerr << "Not writing nbest lists because no out dir is specified\n";
    return false;
  }
}

// return a string consisting of all hypotheses words
char *
LatticeNBestHyp::getHypFeature(SubVocab &ignoreWords, Lattice &lat,
						const char *multiwordSeparator)
{
    unsigned featureLen = 0;
    LatticeNBestPath *prev;

    for (prev = nbestPath; prev != 0; prev = prev->pred) {
        LatticeNode *node = lat.findNode(prev->node);
	assert(node != 0);

        if (!lat.ignoreWord(node->word) && !ignoreWords.getWord(node->word)) {
	    featureLen += strlen(lat.getWord(node->word)) + 1;
        }           
    }

    char *feature = (char *)malloc(featureLen + 1);
    assert(feature != 0);
    feature[featureLen] = '\0';

    for (prev = nbestPath; prev != 0; prev = prev->pred) {
        LatticeNode *node = lat.findNode(prev->node);

        if (!lat.ignoreWord(node->word) && !ignoreWords.getWord(node->word)) {
	    unsigned wordLen = strlen(lat.getWord(node->word));

	    featureLen -= wordLen + 1;

	    strcpy(&feature[featureLen], lat.getWord(node->word));
	    feature[featureLen + wordLen] =
				multiwordSeparator ? *multiwordSeparator : ' ';
        }           
    }

    return feature;
}