dict.c

来自「WinCE平台上的语音识别程序」· C语言 代码 · 共 1,292 行 · 第 1/3 页

        free(entry);
    }

    free(dict->ci_index);
    hash_table_free(dict->dict);
    free(dict);
}

static void
dict_load(dictT * dict, char *filename, int32 * word_id, int32 use_context)
{
    static char const *rname = "dict_load";
    char dict_str[1024];
    char pronoun_str[1024];
    dict_entry_t *entry;
    FILE *fs;
    int32 start_wid = *word_id;
    int32 err = 0;

    fs = myfopen(filename, "r");

    fscanf(fs, "%s\n", dict_str);
    if (strcmp(dict_str, "!") != 0) {
        E_INFO("%s: first line of %s was %s, expecting '!'\n",
               rname, filename, dict_str);
        E_INFO("%s: will assume first line contains a word\n", rname);
        fseek(fs, 0, SEEK_SET);
    }

    pronoun_str[0] = '\0';
    while (EOF != fscanf(fs, "%s%[^\n]\n", dict_str, pronoun_str)) {
        entry = _new_dict_entry(dict_str, pronoun_str, use_context);
        if (!entry) {
            E_ERROR("Failed to add %s to dictionary\n", dict_str);
            err = 1;
            continue;
        }

        _dict_list_add(dict, entry);
        hash_table_enter(dict->dict, entry->word, (void *) *word_id);
        entry->wid = *word_id;
        entry->fwid = *word_id;
        entry->lm_pprob = 0;
        pronoun_str[0] = '\0';
        /*
         * Look for words of the form ".*(#)". These words are
         * alternate pronunciations. Also look for phrases
         * concatenated with '_'.
         */
        {
            char *p = strrchr(dict_str, '(');

            /*
             * For alternate pron. the last car of the word must be ')'
             * This will handle the case where the word is something like
             * "(LEFT_PAREN"
             */
            if (dict_str[strlen(dict_str) - 1] != ')')
                p = NULL;

            if (p != NULL) {
                void *wid;

                if (p)
                    *p = '\0';

                if (hash_table_lookup(dict->dict, dict_str, &wid) != 0) {
                    E_FATAL
                        ("%s: Missing first pronunciation for [%s]\nThis means that e.g. [%s(2)] was found with no [%s]\nPlease correct the dictionary and re-run.\n",
                         rname, dict_str, dict_str, dict_str);
                    exit(1);
                }
                DFPRINTF((stdout,
                          "Alternate transcription for [%s](wid = %d)\n",
                          entry->word, (int32) wid));
                entry->wid = (int32) wid;
                entry->fwid = (int32) wid;
                {
                    while (dict->dict_list[(int32) wid]->alt >= 0)
                        wid = (void *) dict->dict_list[(int32) wid]->alt;
                    dict->dict_list[(int32) wid]->alt = *word_id;
                }
            }
        }

        *word_id = *word_id + 1;
    }

    E_INFO("%6d = words in file [%s]\n", *word_id - start_wid, filename);

    if (fs)
        fclose(fs);

    if (err) {
        E_FATAL("Dictionary errors; cannot continue\n");
    }
}

int32
dictStrToWordId(dictT * dict, char const *dict_str, int verbose)
/*------------------------------------------------------------*
 * return the dict id for dict_str
 *------------------------------------------------------------*/
{
    static char const *rname = "dict_to_id";
    void * dict_id;

    if (hash_table_lookup(dict->dict, dict_str, &dict_id)) {
        if (verbose)
            fprintf(stderr, "%s: did not find %s\n", rname, dict_str);
        return NO_WORD;
    }

    return (int32)dict_id;
}

int32
dict_to_id(dictT * dict, char const *dict_str)
{
    return ((int32) dictStrToWordId(dict, dict_str, FALSE));
}

char const *
dictid_to_str(dictT * dict, int32 id)
{
    return (dict->dict_list[id]->word);
}

static dict_entry_t *
_new_dict_entry(char *word_str, char *pronoun_str, int32 use_context)
{
    dict_entry_t *entry;
    char *phone[MAX_PRONOUN_LEN];
    int32 ciPhoneId[MAX_PRONOUN_LEN];
    int32 triphone_ids[MAX_PRONOUN_LEN];
    int32 pronoun_len = 0;
    int32 i;
    int32 lcTabId;
    int32 rcTabId;
    char triphoneStr[80];
    char position[256];         /* phone position */

    memset(position, 0, sizeof(position));      /* zero out the position matrix */

    position[0] = 'b';          /* First phone is at begginging */

    while (1) {
	if (pronoun_len >= MAX_PRONOUN_LEN) {
	    E_ERROR("'%s': Too many phones for bogus hard-coded limit (%d), skipping\n",
		    word_str, MAX_PRONOUN_LEN);
	    return NULL;
	}
        phone[pronoun_len] = (char *) nxtarg(&pronoun_str, " \t");
        if (*phone[pronoun_len] == 0)
            break;
        /*
         * An '&' in the phone string indicates that this is a word break and
         * and that the previous phone is in the end of word position and the
         * next phone is the begining of word position
         */
        if (phone[pronoun_len][0] == '&') {
            position[pronoun_len - 1] = 'e';
            position[pronoun_len] = 'b';
            continue;
        }
        ciPhoneId[pronoun_len] = phone_to_id(phone[pronoun_len], TRUE);
        if (ciPhoneId[pronoun_len] == NO_PHONE) {
            E_ERROR("'%s': Unknown phone '%s'\n", word_str,
                    phone[pronoun_len]);
            return NULL;
        }
        pronoun_len++;
    }

    position[pronoun_len - 1] = 'e';    /* Last phone is at the end */

    /*
     * If the position marker sequence 'ee' appears or 'se' appears
     * the sequence should be '*s'.
     */

    if (position[0] == 'e')     /* Also handle single phone word case */
        position[0] = 's';

    for (i = 0; i < pronoun_len - 1; i++) {
        if (((position[i] == 'e') || (position[i] == 's')) &&
            (position[i + 1] == 'e'))
            position[i + 1] = 's';
    }

    if (pronoun_len >= 2) {
        i = 0;

        if (use_context) {
            sprintf(triphoneStr, "%s(%%s,%s)b", phone[i], phone[i + 1]);
            lcTabId = addToLeftContextTable(triphoneStr);
            triphone_ids[i] = lcTabId;
        }
        else {
            sprintf(triphoneStr, "%s(%s,%s)b", phone[i], "%s",
                    phone[i + 1]);
            triphone_ids[i] = phone_to_id(triphoneStr, FALSE);
            if (triphone_ids[i] < 0) {
                triphone_ids[i] = phone_to_id(phone[i], TRUE);
                recordMissingTriphone(triphoneStr);
            }
            triphone_ids[i] = bin_mdef_pid2ssid(mdef, phone_map(triphone_ids[i]));
        }

        for (i = 1; i < pronoun_len - 1; i++) {
            sprintf(triphoneStr, "%s(%s,%s)%c",
                    phone[i], phone[i - 1], phone[i + 1], position[i]);
            triphone_ids[i] = phone_to_id(triphoneStr, FALSE);
            if (triphone_ids[i] < 0) {
                triphone_ids[i] = phone_to_id(phone[i], TRUE);
                recordMissingTriphone(triphoneStr);
            }
            triphone_ids[i] = bin_mdef_pid2ssid(mdef,triphone_ids[i]);
        }

        if (use_context) {
            sprintf(triphoneStr, "%s(%s,%%s)e", phone[i], phone[i - 1]);
            rcTabId = addToRightContextTable(triphoneStr);
            triphone_ids[i] = rcTabId;
        }
        else {
            sprintf(triphoneStr, "%s(%s,%s)e", phone[i], phone[i - 1],
                    "%s");
            triphone_ids[i] = phone_to_id(triphoneStr, FALSE);
            if (triphone_ids[i] < 0) {
                triphone_ids[i] = phone_to_id(phone[i], TRUE);
                recordMissingTriphone(triphoneStr);
            }
            triphone_ids[i] = bin_mdef_pid2ssid(mdef,phone_map(triphone_ids[i]));
        }
    }

    /*
     * It's too hard to model both contexts so I choose to model only
     * the left context.
     */
    if (pronoun_len == 1) {
        if (use_context) {
            sprintf(triphoneStr, "%s(%%s,SIL)s", phone[0]);
            lcTabId = addToLeftContextTable(triphoneStr);
            triphone_ids[0] = lcTabId;
            /*
             * Put the right context table in the 2 entry
             */
            sprintf(triphoneStr, "%s(SIL,%%s)s", phone[0]);
            rcTabId = addToRightContextTable(triphoneStr);
            triphone_ids[1] = rcTabId;
        }
        else {
            sprintf(triphoneStr, "%s(%s,%s)s", phone[0], "%s", "%s");
            triphone_ids[0] = phone_to_id(triphoneStr, FALSE);
            if (triphone_ids[0] < 0) {
                triphone_ids[0] = phone_to_id(phone[0], TRUE);
            }
            triphone_ids[i] = bin_mdef_pid2ssid(mdef,triphone_ids[i]);
        }
    }

    entry = (dict_entry_t *) calloc((size_t) 1, sizeof(dict_entry_t));
    entry->word = ckd_salloc(word_str);
    entry->len = pronoun_len;
    entry->mpx = use_context;
    entry->alt = -1;
    if (pronoun_len != 0) {
        entry->ci_phone_ids =
            (int32 *) calloc((size_t) pronoun_len, sizeof(int32));
        memcpy(entry->ci_phone_ids, ciPhoneId,
               pronoun_len * sizeof(int32));
        /*
         * This is a HACK to handle the left right conflict on
         * single phone words
         */
        if (use_context && (pronoun_len == 1))
            pronoun_len += 1;
        entry->phone_ids =
            (int32 *) calloc((size_t) pronoun_len, sizeof(int32));
        memcpy(entry->phone_ids, triphone_ids,
               pronoun_len * sizeof(int32));
    }
    else {
        E_WARN("%s has no pronounciation, will treat as dummy word\n",
               word_str);
    }

    return (entry);
}

/*
 * Replace an existing dictionary entry with the given one.  The existing entry
 * might be a dummy placeholder for new OOV words, in which case the new_entry argument
 * would be TRUE.  (Some restrictions at this time on words that can be added; the code
 * should be self-explanatory.)
 * Return 1 if successful, 0 if not.
 */
static int32
replace_dict_entry(dictT * dict,
                   dict_entry_t * entry,
                   char const *word_str,
                   char *pronoun_str,
                   int32 use_context, int32 new_entry)
{
    char *phone[MAX_PRONOUN_LEN];
    int32 ciPhoneId[MAX_PRONOUN_LEN];
    int32 triphone_ids[MAX_PRONOUN_LEN];
    int32 pronoun_len = 0;
    int32 i;
    char triphoneStr[80];
    void * idx;
    int32 basewid;

    /* For the moment assume left/right context words... */
    assert(use_context);

    /* For the moment, no phrase dictionary stuff... */
    while (1) {
	if (pronoun_len >= MAX_PRONOUN_LEN) {
	    E_ERROR("'%s': Too many phones for bogus hard-coded limit (%d), skipping\n",
		    word_str, MAX_PRONOUN_LEN);
	    return 0;
	}
        phone[pronoun_len] = (char *) nxtarg(&pronoun_str, " \t");
        if (*phone[pronoun_len] == 0)
            break;
        ciPhoneId[pronoun_len] = phone_to_id(phone[pronoun_len], TRUE);
        if (ciPhoneId[pronoun_len] == NO_PHONE) {
            E_ERROR("'%s': Unknown phone '%s'\n", word_str,
                    phone[pronoun_len]);
            return 0;
        }

        pronoun_len++;
    }

    /* For the moment, no single phone new word... */
    if (pronoun_len < 2) {
        E_ERROR("Pronunciation string too short\n");
        return (0);
    }

    /* Check if it's an alternative pronunciation; if so base word must exist */
    {
        char *p = strrchr(word_str, '(');
        if (p && (word_str[strlen(word_str) - 1] == ')')) {
            *p = '\0';
            if (hash_table_lookup(dict->dict, word_str, &idx)) {
                *p = '(';
                E_ERROR("Base word missing for %s\n", word_str);
                return 0;
            }
            *p = '(';
            basewid = (int32) idx;
        }
        else
            basewid = -1;
    }

    /* Parse pron; for the moment, the boundary diphones must be already known... */
    i = 0;
    sprintf(triphoneStr, "%s(%%s,%s)b", phone[i], phone[i + 1]);
    if (hash_table_lookup(lcHT, triphoneStr, &idx) < 0) {
        E_ERROR("Unknown left diphone '%s'\n", triphoneStr);
        return (0);
    }
    triphone_ids[i] = (int32) idx;

    for (i = 1; i < pronoun_len - 1; i++) {
        sprintf(triphoneStr, "%s(%s,%s)", phone[i], phone[i - 1],
                phone[i + 1]);
        triphone_ids[i] = phone_to_id(triphoneStr, FALSE);
        if (triphone_ids[i] < 0)
            triphone_ids[i] = phone_to_id(phone[i], TRUE);
        triphone_ids[i] = bin_mdef_pid2ssid(mdef,triphone_ids[i]);
    }

    sprintf(triphoneStr, "%s(%s,%%s)e", phone[i], phone[i - 1]);
    if (hash_table_lookup(rcHT, triphoneStr, &idx) < 0) {
        E_ERROR("Unknown right diphone '%s'\n", triphoneStr);
        return (0);
    }
    triphone_ids[i] = (int32) idx;

    /*
     * Set up dictionary entry.  Free the existing attributes (where applicable) and
     * replace with new ones.
     */
    entry->len = pronoun_len;
    entry->mpx = use_context;
    free(entry->word);
    free(entry->ci_phone_ids);
    free(entry->phone_ids);
    entry->word = ckd_salloc(word_str);
    entry->ci_phone_ids =
        ckd_calloc((size_t) pronoun_len, sizeof(int32));
    entry->phone_ids =
        ckd_calloc((size_t) pronoun_len, sizeof(int32));
    memcpy(entry->ci_phone_ids, ciPhoneId, pronoun_len * sizeof(int32));
    memcpy(entry->phone_ids, triphone_ids, pronoun_len * sizeof(int32));

    /* Update alternatives linking if adding a new entry (not updating existing one) */
    if (new_entry) {
        entry->alt = -1;
        if (basewid >= 0) {
            entry->alt = dict->dict_list[(int32) basewid]->alt;
            dict->dict_list[(int32) basewid]->alt = entry->wid;
            entry->wid = entry->fwid = (int32) basewid;
        }
    }

    return (1);
}

/*
 * Add a new word to the dictionary, replacing a dummy placeholder.  Or replace an
 * existing non-dummy word in the dictionary.
 * Return the word id of the entry updated if successful.  If any error, return -1.
 */
int32
dict_add_word(dictT * dict, char const *word, char *pron)
{
    dict_entry_t *entry;
    int32 wid, new_entry;

    /* Word already exists */
    new_entry = 0;
    if ((wid = kb_get_word_id(word)) < 0) {
        if (first_dummy > last_dummy) {