tiedstateacousticmodel.java

It is the Speech recognition software. It is platform independent. To execute the source code,

/*
 * Copyright 1999-2002 Carnegie Mellon University.  
 * Portions Copyright 2002 Sun Microsystems, Inc.  
 * Portions Copyright 2002 Mitsubishi Electric Research Laboratories.
 * All Rights Reserved.  Use is subject to license terms.
 * 
 * See the file "license.terms" for information on usage and
 * redistribution of this file, and for a DISCLAIMER OF ALL 
 * WARRANTIES.
 *
 */

package edu.cmu.sphinx.linguist.acoustic.tiedstate;

import java.io.IOException;

import java.util.ArrayList;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;

import edu.cmu.sphinx.linguist.acoustic.AcousticModel;
import edu.cmu.sphinx.linguist.acoustic.Context;
import edu.cmu.sphinx.linguist.acoustic.HMM;
import edu.cmu.sphinx.linguist.acoustic.HMMPosition;
import edu.cmu.sphinx.linguist.acoustic.LeftRightContext;
import edu.cmu.sphinx.linguist.acoustic.Unit;
import edu.cmu.sphinx.linguist.acoustic.UnitManager;

import edu.cmu.sphinx.linguist.acoustic.tiedstate.*;

import edu.cmu.sphinx.util.SphinxProperties;
import edu.cmu.sphinx.util.Timer;
import edu.cmu.sphinx.util.props.Configurable;
import edu.cmu.sphinx.util.props.PropertyException;
import edu.cmu.sphinx.util.props.PropertySheet;
import edu.cmu.sphinx.util.props.PropertyType;
import edu.cmu.sphinx.util.props.Registry;

/**
 * Loads a tied-state acoustic model generated by the Sphinx-3 trainer.
 *
 * <p>
 * It is not the goal of this documentation to provide an explanation 
 * about the concept of HMMs. The explanation below is superficial,
 * and provided only in a way that the files in the acoustic model 
 * package make sense.
 * <p>
 * An HMM models a process using a sequence of states. Associated with 
 * each state, there is a probability density function. A popular choice
 * for this function is a Gaussian mixture, that is, a summation of Gaussians.
 * As you may recall, a single Gaussian is defined by a mean and a variance,
 * or, in the case of a multidimensional Gaussian, by a mean vector and
 * a covariance matrix, or, under some simplifying assumptions,
 * a variance vector. The "means" and "variances" files in the 
 * "continuous" directory contain exactly this: a table in which
 * each line contains a mean vector or a variance vector respectively.
 * The dimension of these vectors is the same as the incoming data,
 * the encoded speech signal. The Gaussian mixture is a summation of
 * Gaussians, with different weights for different Gaussians.
 * The "mixture_weights" file contains this: each line contains
 * the weights for a combination of Gaussians.
 * <p>
 * The HMM is a model with a set of states. The transitions between states
 * have an associated probability. These probabilities make up the
 * transition matrices stored in the "transition_matrices" file.
 * <p>
 * The files in the "continuous" directory are, therefore, tables, or pools,
 * of means, variances, mixture weights, and transition probabilities.
 * <p>
 * The dictionary is a file that maps words to their phonetic transcriptions,
 * that is, it maps words to sequences of phonemes.
 * <p>
 * The language model contains information about probabilities of words 
 * in a language. These probabilities could be for individual words
 * or for sequences of two or three words.
 * <p>
 * The model definition file in a way ties everything together. 
 * If the recognition system models phonemes, there is an HMM for 
 * each phoneme. The model definition file has one line for each phoneme. 
 * The phoneme could be in a context dependent or independent.
 * Each line, therefore, identifies a unique HMM. This line has the 
 * phoneme identification, the non-required left or right context,
 * the index of a transition matrix, and, for each state,
 * the index of a mean vector, a variance vector, and a set of mixture weights.
 *
 */
public class TiedStateAcousticModel implements AcousticModel, Configurable {

    /**
     * The property that defines the component used to load the acoustic model
     */    
    public final static String PROP_LOADER = "loader";

    /**
     * The property that defines the unit manager 
     */    
    public final static String PROP_UNIT_MANAGER  = "unitManager";
    
    /**
     * Controls whether we generate composites or CI units when no
     * context is given during a lookup.
     */
    public final static String PROP_USE_COMPOSITES =  "useComposites";

    /**
     * The default value of PROP_USE_COMPOSITES.
     */
    public final static boolean PROP_USE_COMPOSITES_DEFAULT = true;



    /**
     * Model load timer
     */
    protected final static String TIMER_LOAD = "AM_Load";

    
    // -----------------------------
    // Configured variables
    // -----------------------------
    protected String name;
    private  Logger logger;
    protected Loader loader;
    protected UnitManager unitManager;
    private boolean useComposites = false;
    private Properties properties;    

    // ----------------------------
    // internal variables
    // -----------------------------
    transient protected Timer loadTimer;
    transient private Map compositeSenoneSequenceCache = new HashMap();
    private boolean allocated = false;


    /* (non-Javadoc)
     * @see edu.cmu.sphinx.util.props.Configurable#register(java.lang.String, edu.cmu.sphinx.util.props.Registry)
     */
    public void register(String name, Registry registry) 
                        throws PropertyException {
        this.name = name;
        registry.register(PROP_LOADER, PropertyType.COMPONENT);
        registry.register(PROP_UNIT_MANAGER, PropertyType.COMPONENT);
        registry.register(PROP_USE_COMPOSITES, PropertyType.BOOLEAN);
    }

    /* (non-Javadoc)
     * @see edu.cmu.sphinx.util.props.Configurable#newProperties(edu.cmu.sphinx.util.props.PropertySheet)
     */
    public void newProperties(PropertySheet ps) throws PropertyException {
        loader = (Loader) ps.getComponent(PROP_LOADER, Loader.class);
        unitManager = (UnitManager) ps.getComponent(PROP_UNIT_MANAGER,
                UnitManager.class);
        useComposites = 
            ps.getBoolean(PROP_USE_COMPOSITES, PROP_USE_COMPOSITES_DEFAULT);
        logger = ps.getLogger();
    }

    /**
     * initialize this acoustic model with the given name and context.
     *
     * @throws IOException if the model could not be loaded
     *
     */
    public void allocate() throws IOException {
	if (!allocated) {
	    this.loadTimer = Timer.getTimer(TIMER_LOAD);
	    loadTimer.start();
	    loader.load();
	    loadTimer.stop();
	    logInfo();
	    allocated = true;
	}
    }
    
    /* (non-Javadoc)
     * @see edu.cmu.sphinx.linguist.acoustic.AcousticModel#deallocate()
     */
    public void deallocate() {
    }

    /**
     * Returns the name of this AcousticModel, or null if it has no name.
     *
     * @return the name of this AcousticModel, or null if it has no name
     */
    public String getName() {
	return name;
    }
    

     /**
      * Gets a composite HMM for the given unit and context
      *
      * @param unit the unit for the hmm
      * @param position the position of the unit within the word
      *
      * @return a composite HMM
      */
     private HMM getCompositeHMM(Unit unit, HMMPosition position) {


         if (true) { // use a true composite
             Unit ciUnit = unitManager.getUnit(unit.getName(),
                     unit.isFiller(), Context.EMPTY_CONTEXT);

             SenoneSequence compositeSequence =
                 getCompositeSenoneSequence(unit, position);

             SenoneHMM contextIndependentHMM = (SenoneHMM) 
                 lookupNearestHMM(ciUnit,
                     HMMPosition.UNDEFINED, true);
             float[][] tmat = contextIndependentHMM.getTransitionMatrix();
             return new SenoneHMM(unit, compositeSequence, tmat, position);
         } else { // BUG: just a test. use CI units instead of composites
             Unit ciUnit = lookupUnit(unit.getName());

             assert unit.isContextDependent();
             if (ciUnit == null) {
                 logger.severe("Can't find HMM for " + unit.getName());
             }
             assert ciUnit != null;
             assert !ciUnit.isContextDependent();

	     HMMManager mgr = loader.getHMMManager();
             HMM hmm = mgr.get(HMMPosition.UNDEFINED, ciUnit);
             return hmm;
        }
     }

     /**
      * Given a unit, returns the HMM that best matches the given unit.
      * If exactMatch is false and an exact match is not found, 
      * then different word positions
      * are used. If any of the contexts are non-silence filler units.
      * a silence filler unit is tried instead.
      *
      * @param unit 		the unit of interest
      * @param position 	the position of the unit of interest
      * @param exactMatch 	if true, only an exact match is
      * 			acceptable.
      *
      * @return 	the HMM that best matches, or null if no match
      * 		could be found.
      */
     public HMM lookupNearestHMM(Unit unit, HMMPosition position,
	     boolean exactMatch) {

	 if (exactMatch) {
	     return lookupHMM(unit, position);
	 } else {
	     HMMManager mgr = loader.getHMMManager();
	     HMM hmm = mgr.get(position, unit);

	     if (hmm != null) {
		 return hmm;
	     }
	     // no match, try a composite

	     if (useComposites && hmm == null) {
		 if (isComposite(unit)) {

		     hmm = getCompositeHMM(unit, position);
		     if (hmm != null) {
			 mgr.put(hmm);
		     }
		 }
	     }
	     // no match, try at other positions
	     if (hmm == null) {
		 hmm = getHMMAtAnyPosition(unit);
	     }
	     // still no match, try different filler 
	     if (hmm == null) {
		 hmm = getHMMInSilenceContext(unit, position);
	     }

	     // still no match, backoff to base phone
	     if (hmm == null) {
		 Unit ciUnit = lookupUnit(unit.getName());

		 assert unit.isContextDependent();
                 if (ciUnit == null) {
                     logger.severe("Can't find HMM for " + unit.getName());
                 }
		 assert ciUnit != null;
		 assert !ciUnit.isContextDependent();

		 hmm = mgr.get(HMMPosition.UNDEFINED, ciUnit);
	     }

	     assert hmm != null;

	 // System.out.println("PROX match for "  
	 // 	+ unit + " at " + position + ":" + hmm);

	     return hmm;
	}
     }

     /**
      * Determines if a unit is a composite unit
      *
      * @param unit the unit to test
      *
      * @return true if the unit is missing a right context
      */
     private boolean isComposite(Unit unit) {

	 if (unit.isFiller()) {
	     return false;
	 }

	 Context context = unit.getContext();
	 if (context instanceof LeftRightContext) {
	     LeftRightContext lrContext = (LeftRightContext) context;
	     if (lrContext.getRightContext() == null) {
		 return true;
	     }
	     if (lrContext.getLeftContext() == null) {
		 return true;
	     }
	 }
	 return false;
     }

     /**
      * Looks up the context independent unit given
      * the name
      *
      * @param name the name of the unit
      *
      * @return the unit or null if the unit was not found
      */
     private Unit lookupUnit(String name) {
	 return (Unit) loader.getContextIndependentUnits().get(name);
     }