ngram_read_arpa.c

来自「julius version 4.12.about sound recognit」· C语言 代码 · 共 707 行 · 第 1/2 页

    t->bgn_upper = (NNID_UPPER *)mymalloc_big(sizeof(NNID_UPPER), t->bgnlistlen);
    t->bgn_lower = (NNID_LOWER *)mymalloc_big(sizeof(NNID_LOWER), t->bgnlistlen);
    for(i = 0; i < t->bgnlistlen; i++) {    
      t->bgn_upper[i] = NNID_INVALID_UPPER;
      t->bgn_lower[i] = 0;
    }
  } else {
    t->bgn = (NNID *)mymalloc_big(sizeof(NNID), t->bgnlistlen);
    for(i = 0;i < t->bgnlistlen; i++) {
      t->bgn[i] = NNID_INVALID;
    }
  }
  t->num = (WORD_ID *)mymalloc_big(sizeof(WORD_ID), t->bgnlistlen);
  for(i = 0; i < t->bgnlistlen; i++) {
    t->num[i] = 0;
  }

  /* allocate data area */
  t->nnid2wid = (WORD_ID *)mymalloc_big(sizeof(WORD_ID), t->totalnum);
  t->prob = (LOGPROB *)mymalloc_big(sizeof(LOGPROB), t->totalnum);
  t->bo_wt = NULL;
  t->nnid2ctid_upper = NULL;
  t->nnid2ctid_lower = NULL;

  nnid = 0;
  cid = cid_last = NNID_INVALID;
  for(i=0;i<n;i++) w_last[i] = WORD_INVALID;

  /* read in N-gram */
  for (;;) {
    if (getl(buf, sizeof(buf), fp) == NULL || buf[0] == '\\') break;
    strcpy(pbuf, buf);
    if ( nnid % 100000 == 0) {
      jlog("Stat: ngram_read_arpa: %d-gram read %d (%d%%)\n", n, nnid, nnid * 100 / t->totalnum);
    }

    /* N-gram probability */
    if ((s = strtok(buf, DELM)) == NULL) {
      jlog("Error: ngram_read_arpa: %d-gram: failed to parse, corrupted or invalid data?\n", n);
      return FALSE;
    }
    p = (LOGPROB)atof(s);
    /* read in context word and lookup the ID */
    for(i=0;i<n;i++) {
      if ((s = strtok(NULL, DELM)) == NULL) {
	jlog("Error: ngram_read_arpa: %d-gram: failed to parse, corrupted or invalid data?\n", n);
	return FALSE;
      }
      if ((w[i] = ngram_lookup_word(ndata, s)) == WORD_INVALID) {
	jlog("Error: ngram_read_arpa: %d-gram #%d: \"%s\": \"%s\" not exist in %d-gram\n", n, nnid+1, pbuf, s, n);
	ok_p = FALSE;
	break;
      }
      /* increment nnid_bgn and nnid_num if context word changed */
    }
    if (i < n) continue;	/* error out */

    /* detect context entry change at this line */
    for(i=0;i<n-1;i++) {
      if (w[i] != w_last[i]) break;
    }
    if (i < n-1) {		/* context changed here */
      /* find new entry point */
      cid = search_ngram(ndata, n-1, w);
      if (cid == NNID_INVALID) {	/* no context */
        //jlog("Warning: ngram_read_arpa: %d-gram #%d: \"%s\": context (%s,%s) not exist in %d-gram (ignored)\n", n, nnid+1, pbuf, ndata->wname[w_m], ndata->wname[w_r], n-1);
        jlog("Warning: ngram_read_arpa: %d-gram #%d: \"%s\": context (",
	     n, nnid+1, pbuf);
	for(i=0;i<n-1;i++) {
	  jlog(" %s", ndata->wname[w[i]]);
	}
        jlog(") not exist in %d-gram (ignored)\n", n-1);
	ok_p = FALSE;
        continue;
      }
      if (cid_last != NNID_INVALID) {
	/* close last entry */
	if (t->is24bit) {
	  ntmp = ((NNID)(t->bgn_upper[cid_last]) << 16) + (NNID)(t->bgn_lower[cid_last]);
	} else {
	  ntmp = t->bgn[cid_last];
	}
	t->num[cid_last] = nnid - ntmp;
      }
      /* the next context word should be an new entry */
      if (t->is24bit) {
	if (t->bgn_upper[cid] != NNID_INVALID_UPPER) {
	  jlog("Error: ngram_read_arpa: %d-gram #%d: \"%s\": word order is not the same as 1-gram\n", n, nnid+1, pbuf);
	  return FALSE;
	}
	ntmp = nnid & 0xffff;
	t->bgn_lower[cid] = ntmp;
	ntmp = nnid >> 16;
	t->bgn_upper[cid] = ntmp;
      } else {
	if (t->bgn[cid] != NNID_INVALID) {
	  jlog("Error: ngram_read_arpa: %d-gram #%d: \"%s\": word order is not the same as 1-gram\n", n, nnid+1, pbuf);
	  return FALSE;
	}
	t->bgn[cid] = nnid;
      }

      cid_last = cid;
      w_last[n-1] = WORD_INVALID;
    }

    /* store the probabilities of the target word */
    if (w[n-1] == w_last[n-1]) {
      jlog("Error: ngram_read_arpa: %d-gram #%d: \"%s\": duplicated entry\n", n, nnid+1, pbuf);
      ok_p = FALSE;
      continue;
    } else if (w_last[n-1] != WORD_INVALID && w[n-1] < w_last[n-1]) {
      jlog("Error: ngram_read_arpa: %d-gram #%d: \"%s\": word order is not the same as 1-gram\n", n, nnid+1, pbuf);
      return FALSE;
    }

    /* if the 2-gram has back-off entries, store them here */
    if ((s = strtok(NULL, DELM)) != NULL) {
      bowt = (LOGPROB) atof(s);
      if (t->bo_wt == NULL) {
	t->bo_wt = (LOGPROB *)mymalloc_big(sizeof(LOGPROB), t->totalnum);
	for(i=0;i<nnid;i++) t->bo_wt[i] = 0.0;
      }
      t->bo_wt[nnid] = bowt;
    } else {
      if (t->bo_wt != NULL) t->bo_wt[nnid] = 0.0;
    }

    /* store the entry info */
    t->nnid2wid[nnid] = w[n-1];
    t->prob[nnid] = p;

    nnid++;
    for(i=0;i<n;i++) w_last[i] = w[i];

    /* check total num */
    if (nnid > t->totalnum) {
      jlog("Error: ngram_read_arpa: %d-gram: read num (%d) not match the header value (%d)\n", n, nnid, t->totalnum);
      return FALSE;
    }
  }
  
  /* set the last entry */
  if (t->is24bit) {
    ntmp = ((NNID)(t->bgn_upper[cid_last]) << 16) + (NNID)(t->bgn_lower[cid_last]);
  } else {
    ntmp = t->bgn[cid_last];
  }
  t->num[cid_last] = nnid - ntmp;

  if (t->bo_wt != NULL) t->context_num = t->totalnum;

  if (ok_p == TRUE) {
    jlog("Stat: ngram_read_arpa: %d-gram read %d end\n", n, nnid);
  }

  return ok_p;
}

/** 
 * Read in one ARPA N-gram file.  Supported combinations are
 * LR 2-gram, RL 3-gram and LR 3-gram.
 * 
 * @param fp [in] file pointer
 * @param ndata [out] N-gram data to store the read data
 * @param addition [in] TRUE if going to read additional 2-gram
 * 
 * @return TRUE on success, FALSE on failure.
 */
boolean
ngram_read_arpa(FILE *fp, NGRAM_INFO *ndata, boolean addition)
{
  int i, n;
  NNID num[MAX_N];

  /* source file is not a binary N-gram */
  ndata->from_bin = FALSE;
  ndata->bigram_index_reversed = FALSE;

  /* read until `\data\' found */
  while (getl(buf, sizeof(buf), fp) != NULL && strncmp(buf,"\\data\\",6) != 0);


  if (addition) {
    /* reading additional forward 2-gram for the 1st pass */
    /* read n-gram total info */
    n = get_total_info(fp, num);
    if (n == -1) {		/* error */
      return FALSE;
    }

    /* check N limit */
    if (n < 2) {
      jlog("Error: forward N-gram for pass1 is does not contain 2-gram\n");
      return FALSE;
    }
    if (n > 2) {
      jlog("Warning: forward N-gram for pass1 contains %d-gram, only 2-gram will be used\n", n);
    }

    /* check if the numbers are the same with already read n-gram */
    for(i=0;i<2;i++) {
      if (ndata->d[i].totalnum != num[i]) {
	jlog("Warning: ngram_read_arpa: %d-gram total num differ between forward N-gram and backward N-gram, may cause some error\n", i+1);
      }
    }
    /* read additional 1-gram data */
    if (!strnmatch(buf,"\\1-grams",8)) {
      jlog("Error: ngram_read_arpa: 1-gram not found for additional LR 2-gram\n");
      return FALSE;
    }
    jlog("Stat: ngram_read_arpa: reading 1-gram part...\n");
    if (add_unigram(fp, ndata) == FALSE) return FALSE;
    /* read 2-gram data */
    if (!strnmatch(buf,"\\2-grams", 8)) {
      jlog("Error: ngram_read_arpa: 2-gram not found for additional LR 2-gram\n");
      return FALSE;
    }
    jlog("Stat: ngram_read_arpa: reading 2-gram part...\n");
    if (add_bigram(fp, ndata) == FALSE) return FALSE;


    /* ignore the rest */
    if (strnmatch(buf,"\\3-grams", 8)) {
      jlog("Warning: forward n-gram contains more than 3-gram, ignored\n");
    }

  } else {
    /* read n-gram total info */
    n = get_total_info(fp, num);
    if (n == -1) {		/* error */
      return FALSE;
    }
    jlog("Stat: ngram_read_arpa: this is %d-gram file\n", n);
    for(i=0;i<n;i++) {
      ndata->d[i].totalnum = num[i];
    }
    
    /* set word num */
    if (ndata->d[0].totalnum > MAX_WORD_NUM) {
      jlog("Error: ngram_read_arpa: N-gram vocabulary size exceeds the limit (%d)\n", MAX_WORD_NUM);
      return FALSE;
    }
    ndata->max_word_num = ndata->d[0].totalnum;
    
    /* check if each N-gram allows 24bit and back-off compaction mode */
    /* for fast access, 1-gram and 2-gram always use non-compaction mode */
    for(i=0;i<n;i++) {
      if (i < 2) {		/* not use for 1-gram and 2-gram */
	ndata->d[i].is24bit = FALSE;
      } else {
	/* for 3-gram and later 24 bit mode is preferred,
	   but should be disabled if number of entries is over 2^24 */
	if (ndata->d[i].totalnum > NNID_MAX_24) {
	  jlog("Warning: ngram_read_arpa: num of %d-gram exceeds 24bit, now switch to %dbit index\n", i+1, sizeof(NNID) * 8);
	  ndata->d[i].is24bit = FALSE;
	} else {
	  ndata->d[i].is24bit = TRUE;
	}
      }
    }
    /* disable ct_compaction flag while reading ARPA data */
    for(i=0;i<n;i++) {
      ndata->d[i].ct_compaction = FALSE;
    }
    
    /* read 1-gram data */
    if (!strnmatch(buf,"\\1-grams",8)) {
      jlog("Error: ngram_read_arpa: data format error: 1-gram not found\n");
      return FALSE;
    }
    jlog("Stat: ngram_read_arpa: reading 1-gram part...\n");
    if (set_unigram(fp, ndata) == FALSE) return FALSE;
    
    i = 2;
    while(i <= n) {
      /* read n-gram data in turn */
      sprintf(pbuf, "\\%d-grams", i);
      if (!strnmatch(buf, pbuf, 8)) {
	jlog("Error: ngram_read_arpa: data format error: %d-gram not found\n", i);
	return FALSE;
      }
      jlog("Stat: ngram_read_arpa: reading %d-gram part...\n", i);
      if (set_ngram(fp, ndata, i) == FALSE) return FALSE;
      i++;
    }
    /* finished reading file */
    if (!strnmatch(buf, "\\end", 4)) {
      jlog("Error: ngram_read_arpa: data format error: end marker \"\\end\" not found\n");
      return FALSE;
    }

    ndata->n = n;

    for(i=2;i<n;i++) {
      if (ndata->d[i-1].bo_wt != NULL) {
	/* perform back-off compaction */
	if (ngram_compact_context(ndata, i) == FALSE) return FALSE;
      }
    }
    
    /* swap <s> and </s> for backward SRILM N-gram */
    if (ndata->dir == DIR_RL) {
      WORD_ID bos, eos;
      char *p;
      bos = ngram_lookup_word(ndata, BEGIN_WORD_DEFAULT);
      eos = ngram_lookup_word(ndata, END_WORD_DEFAULT);
      if (!ndata->bos_eos_swap) {
	/* check */
	if (bos != WORD_INVALID && eos != WORD_INVALID && ndata->d[0].prob[bos] == -99) {
	  jlog("Stat: \"P(%s) = -99\" in reverse N-gram, may be trained by SRILM\n", BEGIN_WORD_DEFAULT);
	  jlog("Stat: going to swap \"%s\" and \"%s\"\n", BEGIN_WORD_DEFAULT, END_WORD_DEFAULT);
	  ndata->bos_eos_swap = TRUE;
	}
      }
      if (ndata->bos_eos_swap) {
	if (bos == WORD_INVALID) {
	  jlog("Error: ngram_read_arpa: try to swap bos/eos but \"%s\" not found in N-gram\n", BEGIN_WORD_DEFAULT);
	}
	if (eos == WORD_INVALID) {
	  jlog("Error: ngram_read_arpa: try to swap bos/eos but \"%s\" not found in N-gram\n", END_WORD_DEFAULT);
	}
	if (bos == WORD_INVALID || eos == WORD_INVALID) {
	  return FALSE;
	}
	/* do swap */
	jlog("Stat: ngram_read_arpa: swap \"%s\" and \"%s\" at backward N-gram\n", BEGIN_WORD_DEFAULT, END_WORD_DEFAULT);
	/* swap name buffer */
	p = ndata->wname[bos];
	ndata->wname[bos] = ndata->wname[eos];
	ndata->wname[eos] = p;
	/* replace index */
	ptree_replace_data(BEGIN_WORD_DEFAULT, eos, ndata->root);
	ptree_replace_data(END_WORD_DEFAULT, bos, ndata->root);
      }
    }

  }
    
#ifdef CLASS_NGRAM
  /* skip in-class word entries (they should be in word dictionary) */
  if (getl(buf, sizeof(buf), fp) != NULL) {
    if (strnmatch(buf, "\\class", 6)) {
      jlog("Stat: ngram_read_arpa: skipping in-class word entries...\n");
    }
  }
#endif

  bi_prob_func_set(ndata);

  return TRUE;
}