m_fusion.c

julius version 4.12.about sound recognition.

}

/** 
 * <EN>
 * @brief  Launch a recognition process instance.
 *
 * This function will create an recognition process instance
 * using the given SEARCH configuration, and launch recognizer for
 * the search.  Then the created instance will be installed to the
 * engine instance.  The sconf should be registered to the global
 * jconf before calling this function.
 *
 * </EN>
 *
 * <JA>
 * @brief 千急借妄インスタンスを惟ち惧げるˉ
 *
 * この簇眶は·涂えられた SEARCH 肋年に骄って 千急借妄インスタンスを栏喇し·
 * 滦炳する不兰千急达を菇蜜しますˉその稿·その栏喇された千急借妄インスタンスは
 * 糠たにエンジンインスタンスに判峡されますˉSEARCH肋年はこの簇眶を
 * 钙ぶ涟にあらかじめ链挛肋年jconfに判峡されている涩妥がありますˉ
 * 
 * </JA>
 * 
 * @param recog [i/o] engine instance
 * @param sconf [in] SEARCH configuration to launch
 * 
 * @return TRUE on success, or FALSE on error.
 *
 * @callgraph
 * @callergraph
 * @ingroup instance
 * 
 */
boolean
j_launch_recognition_instance(Recog *recog, JCONF_SEARCH *sconf)
{
  RecogProcess *p;
  PROCESS_AM *am;
  PROCESS_LM *lm;

  jlog("STAT: composing recognizer instance SR%02d %s (AM%02d %s, LM%02d %s)\n", sconf->id, sconf->name, sconf->amconf->id, sconf->amconf->name, sconf->lmconf->id, sconf->lmconf->name);

  /* allocate recognition instance */
  p = j_recogprocess_new(recog, sconf);

  /* assign corresponding AM instance and LM instance to use */
  for(lm=recog->lmlist;lm;lm=lm->next) {
    if (sconf->lmconf == lm->config) {
      for(am=recog->amlist;am;am=am->next) {
	if (sconf->amconf == am->config) {
	  p->am = am;
	  p->lm = lm;
	}
      }
    }
  }

  if (p->config->sw.triphone_check_flag && p->am->hmminfo->is_triphone) {
    /* go into interactive triphone HMM check mode */
    hmm_check(p);
  }
  
  /******************************************/
  /******** set work area and flags *********/
  /******************************************/

  /* copy values of sub instances for handly access during recognition */
  /* set lm type */
  p->lmtype = p->lm->lmtype;
  p->lmvar  = p->lm->lmvar;
  p->graphout = p->config->graph.enabled;
  
  /* set flag for context dependent handling */
  if (p->config->force_ccd_handling) {
    p->ccd_flag = p->config->ccd_handling;
  } else {
    if (p->am->hmminfo->is_triphone) {
      p->ccd_flag = TRUE;
    } else {
      p->ccd_flag = FALSE;
    }
  }

  /* iwsp prepare */
  if (p->lm->config->enable_iwsp) {
    if (p->am->hmminfo->multipath) {
      /* find short-pause model */
      if (p->am->hmminfo->sp == NULL) {
	jlog("ERROR: iwsp enabled but no short pause model \"%s\" in hmmdefs\n", p->am->config->spmodel_name);
	return FALSE;
      }
      p->am->hmminfo->iwsp_penalty = p->am->config->iwsp_penalty;
    } else {
      jlog("Warning: \"-iwsp\" is supported on multi-path mode, ignored\n");
    }
  }

  /* for short-pause segmentation  */
  if (p->config->successive.enabled) {
    if (p->config->successive.pausemodelname) {
      /* pause model name string specified, divide it and store to p */
      char *s;
      int n;
      p->pass1.pausemodelnames = (char*)mymalloc(strlen(p->config->successive.pausemodelname)+1);
      strcpy(p->pass1.pausemodelnames, p->config->successive.pausemodelname);
      n = 0;
      for (s = strtok(p->pass1.pausemodelnames, " ,"); s; s = strtok(NULL, " ,")) {
	n++;
      }
      p->pass1.pausemodelnum = n;
      p->pass1.pausemodel = (char **)mymalloc(sizeof(char *) * n);
      strcpy(p->pass1.pausemodelnames, p->config->successive.pausemodelname);
      n = 0;
      for (s = strtok(p->pass1.pausemodelnames, " ,"); s; s = strtok(NULL, " ,")) {
	p->pass1.pausemodel[n++] = s;
      }
    } else {
      p->pass1.pausemodel = NULL;
    }
    /* check if pause word exists on dictionary */
    {
      WORD_ID w;
      boolean ok_p;
      ok_p = FALSE;
      for(w=0;w<p->lm->winfo->num;w++) {
	if (is_sil(w, p)) {
	  ok_p = TRUE;
	  break;
	}
      }
      if (!ok_p) {
#ifdef SPSEGMENT_NAIST
	jlog("Error: no pause word in dictionary needed for decoder-based VAD\n");
#else
	jlog("Error: no pause word in dictionary needed for short-pause segmentation\n");
#endif
	jlog("Error: you should have at least one pause word in dictionary\n");
	jlog("Error: you can specify pause model names by \"-pausemodels\"\n");
	return FALSE;
      }
    }
  }

  /**********************************************/
  /******** set model-specific defaults *********/
  /**********************************************/
  if (p->lmtype == LM_PROB) {
    /* set default lm parameter if not specified */
    if (!p->config->lmp.lmp_specified) {
      if (p->am->hmminfo->is_triphone) {
	p->config->lmp.lm_weight = DEFAULT_LM_WEIGHT_TRI_PASS1;
	p->config->lmp.lm_penalty = DEFAULT_LM_PENALTY_TRI_PASS1;
      } else {
	p->config->lmp.lm_weight = DEFAULT_LM_WEIGHT_MONO_PASS1;
	p->config->lmp.lm_penalty = DEFAULT_LM_PENALTY_MONO_PASS1;
      }
    }
    if (!p->config->lmp.lmp2_specified) {
      if (p->am->hmminfo->is_triphone) {
	p->config->lmp.lm_weight2 = DEFAULT_LM_WEIGHT_TRI_PASS2;
	p->config->lmp.lm_penalty2 = DEFAULT_LM_PENALTY_TRI_PASS2;
      } else {
	p->config->lmp.lm_weight2 = DEFAULT_LM_WEIGHT_MONO_PASS2;
	p->config->lmp.lm_penalty2 = DEFAULT_LM_PENALTY_MONO_PASS2;
      }
    }
    if (p->config->lmp.lmp_specified != p->config->lmp.lmp2_specified) {
      jlog("WARNING: m_fusion: only -lmp or -lmp2 specified, LM weights may be unbalanced\n");
    }
  }

  /****************************/
  /******* build wchmm ********/
  /****************************/
  if (p->lmtype == LM_DFA) {
    /* execute generation of global grammar and build of wchmm */
    multigram_build(p); /* some modification occured if return TRUE */
  }

  if (p->lmtype == LM_PROB) {
    /* build wchmm with N-gram */
    p->wchmm = wchmm_new();
    p->wchmm->lmtype = p->lmtype;
    p->wchmm->lmvar  = p->lmvar;
    p->wchmm->ccd_flag = p->ccd_flag;
    p->wchmm->category_tree = FALSE;
    p->wchmm->hmmwrk = &(p->am->hmmwrk);
    /* assign models */
    p->wchmm->ngram = p->lm->ngram;
    if (p->lmvar == LM_NGRAM_USER) {
      /* register LM functions for 1st pass here */
      p->wchmm->uni_prob_user = p->lm->lmfunc.uniprob;
      p->wchmm->bi_prob_user = p->lm->lmfunc.biprob;
    }
    p->wchmm->winfo = p->lm->winfo;
    p->wchmm->hmminfo = p->am->hmminfo;
    if (p->wchmm->category_tree) {
      if (p->config->pass1.old_tree_function_flag) {
	if (build_wchmm(p->wchmm, p->lm->config) == FALSE) {
	  jlog("ERROR: m_fusion: error in bulding wchmm\n");
	  return FALSE;
	}
      } else {
	if (build_wchmm2(p->wchmm, p->lm->config) == FALSE) {
	  jlog("ERROR: m_fusion: error in bulding wchmm\n");
	  return FALSE;
	}
      }
    } else {
      if (build_wchmm2(p->wchmm, p->lm->config) == FALSE) {
	jlog("ERROR: m_fusion: error in bulding wchmm\n");
	return FALSE;
      }
    }

    /* 弹瓢箕 -check でチェックモ〖ドへ */
    if (p->config->sw.wchmm_check_flag) {
      wchmm_check_interactive(p->wchmm);
    }

    /* set beam width */
    /* guess beam width from models, when not specified */
    p->trellis_beam_width = set_beam_width(p->wchmm, p->config->pass1.specified_trellis_beam_width);

    /* initialize cache for factoring */
    max_successor_cache_init(p->wchmm);
  }

  /* backtrellis initialization */
  p->backtrellis = (BACKTRELLIS *)mymalloc(sizeof(BACKTRELLIS));
  bt_init(p->backtrellis);

  jlog("STAT: SR%02d %s composed\n", sconf->id, sconf->name);

  if (sconf->sw.start_inactive) {
    /* start inactive */
    p->active = -1;
  } else {
    /* book activation for the recognition */
    p->active = 1;
  }
  if (p->lmtype == LM_DFA) {
    if (p->lm->winfo == NULL ||
	(p->lmvar == LM_DFA_GRAMMAR && p->lm->dfa == NULL)) {
      /* make this instance inactive */
      p->active = -1;
    }
  }

  return TRUE;
}


/** 
 * <EN>
 * @brief  Combine all loaded models and settings into one engine instance.
 *
 * This function will finalize preparation of recognition:
 * 
 *  - create required MFCC calculation instances,
 *  - create recognition process instance for specified LM/AM combination,
 *  - set model-specific recognition parameters,
 *  - build tree lexicon for each process instance for the 1st pass,
 *  - prepare work area and cache area for recognition,
 *  - initialize some values / work area for frontend processing.
 *
 * After this function, all recognition setup was done and we are ready for
 * start recognition.
 *
 * This should be called after j_jconf_finalize() and j_load_all() has been
 * completed.  You should put the jconf at recog->jconf before calling this
 * function.

 * </EN>
 * <JA>
 * @brief  链てのロ〖ドされたモデルと肋年からエンジンインスタンスを
 * 呵姜菇喇する. 
 *
 * この簇眶は·千急洁洒のための呵姜借妄を乖う. 柒婶では·
 *
 *  - 涩妥な MFCC 纷换インスタンスの栏喇
 *  - 回年された LM/AM の寥からの千急借妄インスタンス栏喇
 *  - モデルに巴赂する千急脱パラメ〖タの肋年
 *  - 妈1パス脱の腾菇陇步辑今を千急借妄インスタンスごとに菇蜜
 *  - 千急借妄脱ワ〖クエリアとキャッシュエリアを澄瘦
 *  - フロントエンド借妄のためのいくつかの猛とワ〖クエリアの澄瘦
 *
 *  を乖う. この簇眶が姜位稿·エンジンインスタンス柒の链てのセットアップ
 *  は姜位し·千急が倡幌できる觉轮となる. 
 *
 *  この簇眶は·j_jconf_finalize() と j_load_all() が姜わった觉轮で
 *  钙び叫す涩妥がある. 钙叫し涟には·recog->jconf に (j_load_all でともに
 *  蝗脱した) jconf を呈羌しておくこと. 
 * 
 * </JA>
 * 
 * @param recog [in] engine instance
 * 
 * @return TRUE when all initialization successfully done, or FALSE if any
 * error has been occured.
 *
 * @callgraph
 * @callergraph
 * @ingroup instance
 * 
 */
boolean
j_final_fusion(Recog *recog)
{
  MFCCCalc *mfcc;
  JCONF_SEARCH *sconf;
  PROCESS_AM *am;

  jlog("STAT: ------\n");
  jlog("STAT: All models are ready, go for final fusion\n");
  jlog("STAT: [1] create MFCC extraction instance(s)\n");
  if (recog->jconf->input.type == INPUT_WAVEFORM) {
    /***************************************************/
    /* create MFCC calculation instance from AM config */
    /* according to the fixated parameter information  */
    /***************************************************/
    create_mfcc_calc_instances(recog);
  }

  /****************************************/
  /* create recognition process instances */
  /****************************************/
  jlog("STAT: [2] create recognition processing instance(s) with AM and LM\n");
  for(sconf=recog->jconf->search_root;sconf;sconf=sconf->next) {
    if (j_launch_recognition_instance(recog, sconf) == FALSE) return FALSE;
  }

  /****************************/
  /****** initialize GMM ******/
  /****************************/
  if (recog->gmm != NULL) {
    jlog("STAT: [2.5] create GMM instance\n");
    if (gmm_init(recog) == FALSE) {
      jlog("ERROR: m_fusion: error in initializing GMM\n");
      return FALSE;
    }
  }

  /* stage 4: setup output probability function for each AM */
  jlog("STAT: [3] initialize for acoustic HMM calculation\n");
  for(am=recog->amlist;am;am=am->next) {
#ifdef ENABLE_PLUGIN
    /* set plugin function if specified */
    if (am->config->gprune_method == GPRUNE_SEL_USER) {
      am->hmmwrk.compute_gaussset = (void (*)(HMMWork *, HTK_HMM_Dens **, int, int *, int)) plugin_get_func(am->config->gprune_plugin_source, "calcmix");
      if (am->hmmwrk.compute_gaussset == NULL) {
	jlog("ERROR: calcmix plugin has no function \"calcmix\"\n");
	return FALSE;
      }
      am->hmmwrk.compute_gaussset_init = (boolean (*)(HMMWork *)) plugin_get_func(am->config->gprune_plugin_source, "calcmix_init");
      if (am->hmmwrk.compute_gaussset_init == NULL) {
	jlog("ERROR: calcmix plugin has no function \"calcmix_init\"\n");
	return FALSE;
      }
      am->hmmwrk.compute_gaussset_free = (void (*)(HMMWork *)) plugin_get_func(am->config->gprune_plugin_source, "calcmix_free");
      if (am->hmmwrk.compute_gaussset_free == NULL) {
	jlog("ERROR: calcmix plugin has no function \"calcmix_free\"\n");
	return FALSE;
      }
    }
#endif
    if (am->config->hmm_gs_filename != NULL) {/* with GMS */
      if (outprob_init(&(am->hmmwrk), am->hmminfo, am->hmm_gs, am->config->gs_statenum, am->config->gprune_method, am->config->mixnum_thres) == FALSE) {
	return FALSE;
      }
    } else {
      if (outprob_init(&(am->hmmwrk), am->hmminfo, NULL, 0, am->config->gprune_method, am->config->mixnum_thres) == FALSE) {
	return FALSE;
      }
    }
  }

  /* stage 5: initialize work area for input and realtime decoding */

  jlog("STAT: [4] prepare MFCC storage(s)\n");
  if (recog->jconf->input.type == INPUT_VECTOR) {
    /* create an MFCC instance for MFCC input */
    /* create new mfcc instance */
    recog->mfcclist = j_mfcccalc_new(NULL);
    recog->mfcclist->id = 1;
    /* assign to the am */
    for(am=recog->amlist;am;am=am->next) {
      am->mfcc = recog->mfcclist;
    }
    if (recog->gmm) recog->gmmmfcc = recog->mfcclist;
  }
  /* allocate parameter holders */
  for(mfcc=recog->mfcclist;mfcc;mfcc=mfcc->next) {
    mfcc->param = new_param();
  }
  
  /* initialize SS calculation work area */
  if (recog->jconf->input.type == INPUT_WAVEFORM) {
    for(mfcc=recog->mfcclist;mfcc;mfcc=mfcc->next) {
      if (mfcc->frontend.sscalc) {
	mfcc->frontend.mfccwrk_ss = WMP_work_new(mfcc->para);
	if (mfcc->frontend.mfccwrk_ss == NULL) {
	  jlog("ERROR: m_fusion: failed to initialize MFCC computation for SS\n");
	  return FALSE;
	}
	if (mfcc->frontend.sscalc_len * recog->jconf->input.sfreq / 1000 < mfcc->para->framesize) {
	  jlog("ERROR: m_fusion: head sil length for SS (%d msec) is shorter than a frame (%d msec)\n", mfcc->frontend.sscalc_len, mfcc->para->framesize * 1000 / recog->jconf->input.sfreq);
	  return FALSE;
	}
      }
    }
  }

  if (recog->jconf->decodeopt.realtime_flag) {
    jlog("STAT: [5] prepare for real-time decoding\n");
    /* prepare for 1st pass pipeline processing */
    if (recog->jconf->input.type == INPUT_WAVEFORM) {
      if (RealTimeInit(recog) == FALSE) {
	jlog("ERROR: m_fusion: failed to initialize recognition process\n");
	return FALSE;
      }
    }
  }

  /* finished! */
  jlog("STAT: All init successfully done\n\n");

  /* set-up callback plugin if any */
#ifdef ENABLE_PLUGIN
  if (plugin_exec_engine_startup(recog) == FALSE) {
    jlog("ERROR: m_fusion: failed to execute callback setup in plugin\n");
    return FALSE;
  }
#endif

  return TRUE;
}

/* end of file */