perfestdispatch.java

机器学习算法中的FSS算法

package fss;

import shared.*;
import shared.Error;
import java.lang.*;
import java.io.*;

//PerfEstDispatch.c is in MWrapper.

public class PerfEstDispatch{
/* ErrorType ENUM */
   public final static int classError = 0;
   public final static int meanSquaredError = 1;
   public final static int meanAbsoluteError = 2;
   public final static int classLoss = 3;
/* END ENUM */

   // note: PerfEstimationMethod is public but must be declared BEFORE
   // perfEstimationMethod, which is private.  Hence, the violation of
   // ordering rules here.

/* PerfEstimationMethod ENUM */
   public final static int cv = 0;
   public final static int stratCV = 1;
   public final static int testSet = 2;
   public final static int bootstrap = 3;
   public final static int holdOut = 4;
   public final static int automatic = 5;
/* END ENUM */

/**  **/
   protected int DEFAULT_PERF_ESTIMATOR = automatic; //PerfEstimationMethod enum
   protected int DEFAULT_ERROR_TYPE = classError;  //ErrorType enum
   protected double DEFAULT_CV_WEIGHT_THRESHOLD = 5000;
   protected int DEFAULT_SEED = 7258789;
   protected int DEFAULT_CV_FOLDS = 10;
   protected int DEFAULT_CV_TIMES = 1;
   protected double DEFAULT_CV_FRACT = 1.0;
   protected int DEFAULT_MAX_TIMES = 5;
   protected double DEFAULT_DES_STD_DEV = 0.01;
   protected double DEFAULT_ERROR_TRIM = 0.0;
   protected int DEFAULT_BOOTSTRAP_TIMES = 10;
   protected double DEFAULT_BOOTSTRAP_FRACTION = 0.632;
   protected int DEFAULT_HOLDOUT_TIMES = 1;
   protected int DEFAULT_HOLDOUT_NUMBER = 0;
   protected double DEFAULT_HOLDOUT_PERCENT = 2/3; //Real(2)/3;

// help strings for options
   protected String PERF_ESTIMATION_METHOD_HELP =
      "This option selects the method of performance estimation to use in "
      +"evaluating states.  We also allow use of the real error rate (when "
      +"available) to get an upper bound on performance of the search.";
   protected String ERROR_TYPE_HELP =
      "This option selects the dominant type of error estimate to produce.  "
      +"Error chooses raw classification error rate.  Mean squared error and "
      +"mean absolute error are metrics which attempt to assess the performance "
      +"of probability estimates.  Loss allows the loss matrix (if provided) "
      +"to alter performance estimates.";
   protected String CV_WEIGHT_THRESHOLD_HELP =
      "The instance list total weight threshold below which using the automatic "
      +"performance estimation method will result in cross-validation, and above "
      +"which will result in hold out.";
   protected String SEED_HELP =
      "This option specifies a specific seed for the random number generator.";   
   protected String CV_FOLDS_HELP =
      "This option specifies the number of folds to use for cross-validation."
      +"  Specifying a negative number -k produces the leave-k-out algorithm."
      +"  It is an error to choose zero or one.";
   protected String CV_TIMES_HELP =
      "This option specifies the number of times to run cross-validation.  "
      +"Choosing zero times will cause the program to automatically select "
      +"an appropriate number in order to minimize variance.";
   protected String CV_FRACT_HELP =
      "This option specifies the fraction of the proposed training set to "
      +"use within each fold of cross-validation.  Choosing 1.0 performs "
      +"standard cross-validation.";
   protected String MAX_TIMES_HELP =
      "This option specifies the maximum number of times to try to run "
      +"cross-validation while attempting to achieve the desired standard "
      +"deviation.";
   protected String DES_STD_DEV_HELP =
      "This option specifies the desired standard deviation for an automatic"
      +" run of cross-validation.";
   protected String ERROR_TRIM_HELP =
      "This option specifies the trim used for determining error means.";
   protected String BOOTSTRAP_TIMES_HELP =
      "This option specifies the number of times to run bootstrap.";
   protected String BOOTSTRAP_FRACTION_HELP =
      "This option specifies the weight given to the bootstrap sample in the "
      +"bootstrap formula.  Typical values are 0.632 and 0.5.";
   protected String HOLDOUT_TIMES_HELP =
      "This option specifies the number of times to repeat holdout.";
   protected String HOLDOUT_NUMBER_HELP =
      "This option specifies an exact number of instances to hold out for "
      +"training or testing.  Select a positive number to hold out instances "
      +"for training.  Select a negative number to hold out instances for "
      +"testing.  Select zero to choose a percentage instead.";
   protected String HOLDOUT_PERCENT_HELP =
      "This option specifies a percentage of instances to hold out for "
      +"training.";


   public static MEnum perfEstimationMethodEnum = new MEnum();

   public static MEnum errorTypeEnum;

   double errTrim;
   double stdDevTrim;
   int randSeed;
   double cvWeightThreshold;
   
   // CV options
   int cvFolds;
   int cvTimes;
   int maxTimes;
   double cvFraction;
   double desStdDev;

   // Bootstrap options
   int bootstrapTimes;
   double bootstrapFraction;

   // HoldOut options
   int hoTimes;
   int hoNumber;
   double hoPercent;

   // results
   PerfData perfData = new PerfData();
   int actualTimes;   

   int perfEstimationMethod; //PerfEstimationMethod enum
   int errorType; //ErrorType enum

/** Logging options for this class. **/
   protected LogOptions logOptions = new LogOptions();

/***************************************************************************
  Sets the logging level for this object.
@param level	The new logging level.
***************************************************************************/
   public void set_log_level(int level){logOptions.set_log_level(level);}

/***************************************************************************
  Returns the logging level for this object.
***************************************************************************/
   public int  get_log_level(){return logOptions.get_log_level();}

/***************************************************************************
  Sets the stream to which logging options are displayed.
@param strm	The stream to which logs will be written.
***************************************************************************/
   public void set_log_stream(Writer strm)
      {logOptions.set_log_stream(strm);}

/***************************************************************************
  Returns the stream to which logs for this object are written.
@return The stream to which logs for this object are written.
***************************************************************************/
   public Writer get_log_stream(){return logOptions.get_log_stream();}

/***************************************************************************
  Returns the LogOptions object for this object.
@return The LogOptions object for this object.
***************************************************************************/
   public LogOptions get_log_options(){return logOptions;}

/***************************************************************************
  Sets the LogOptions object for this object.
@param opt	The new LogOptions object.
***************************************************************************/
   public void set_log_options(LogOptions opt)
      {logOptions.set_log_options(opt);}

/***************************************************************************
  Sets the logging message prefix for this object.
@param file	The file name to be displayed in the prefix of log messages.
@param line	The line number to be displayed in the prefix of log messages.
@param lvl1 The log level of the statement being logged.
@param lvl2	The level of log messages being displayed.
***************************************************************************/
   public void set_log_prefixes(String file, int line,int lvl1, int lvl2)
      {logOptions.set_log_prefixes(file, line, lvl1, lvl2);}

/***************************************************************************
  This class has no access to a copy constructor.
***************************************************************************/
   private PerfEstDispatch(PerfEstDispatch source){}

   public PerfEstDispatch() { set_defaults(); }

   public void set_defaults()
   {
      set_perf_estimator(DEFAULT_PERF_ESTIMATOR);
      set_error_type(DEFAULT_ERROR_TYPE);
      set_seed(DEFAULT_SEED);
      set_cv_weight_threshold(DEFAULT_CV_WEIGHT_THRESHOLD);
      set_cv_folds(DEFAULT_CV_FOLDS);
      set_cv_times(DEFAULT_CV_TIMES);
      set_cv_fraction(DEFAULT_CV_FRACT);
      set_max_times(DEFAULT_MAX_TIMES);
      set_desired_std_dev(DEFAULT_DES_STD_DEV);
      set_error_trim(DEFAULT_ERROR_TRIM);
      set_bootstrap_times(DEFAULT_BOOTSTRAP_TIMES);
      set_bootstrap_fraction(DEFAULT_BOOTSTRAP_FRACTION);
      set_holdout_times(DEFAULT_HOLDOUT_TIMES);
      set_holdout_number(DEFAULT_HOLDOUT_NUMBER);
      set_holdout_percent(DEFAULT_HOLDOUT_PERCENT);
   }

/*bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 1
   public void set_user_options(){set_user_options("");}
   public void set_user_options(String prefix)
   {
      set_perf_estimator(
         get_option_enum(
            (prefix + "PERF_ESTIMATOR"), perfEstimationMethodEnum,
            get_perf_estimator(), PERF_ESTIMATION_METHOD_HELP,
            false));
      if (perfEstimationMethod == automatic)
         cvWeightThreshold =
            get_option_real_range(prefix + "CV_WEIGHT_THRESHOLD",
            DEFAULT_CV_WEIGHT_THRESHOLD, 0.0, REAL_MAX,
            CV_WEIGHT_THRESHOLD_HELP, false, false);
      set_error_type(
         get_option_enum(
            (prefix + "ERROR_TYPE"), errorTypeEnum,
            get_error_type(), ERROR_TYPE_HELP, true));
      set_seed(
         get_option_int(prefix + "PERF_EST_SEED", get_seed(),
            SEED_HELP, true));
      set_error_trim(
         get_option_real(prefix + "ERROR_TRIM", get_error_trim(),
            ERROR_TRIM_HELP, true));

      if(perfEstimationMethod == cv || perfEstimationMethod == stratCV ||
         perfEstimationMethod == automatic)
         {
            set_cv_folds(
               get_option_int(prefix + "CV_FOLDS", get_cv_folds(),
                  CV_FOLDS_HELP, false));

            set_cv_times(
               get_option_int(prefix + "CV_TIMES", get_cv_times(),
                  CV_TIMES_HELP, true));

            set_cv_fraction(
               get_option_real_range(prefix + "CV_FRACT", get_cv_fraction(),
                  0.0, 1.0,
                  CV_FRACT_HELP, true));

            if(get_cv_times() == 0)
            {
               set_desired_std_dev(get_option_real(prefix + "DES_STD_DEV",
                  get_desired_std_dev(),
               DES_STD_DEV_HELP, true));
               set_max_times(get_option_int(prefix + "MAX_TIMES", get_max_times(),
                  MAX_TIMES_HELP, true));
            }		   
         }

      if (perfEstimationMethod == bootstrap)
      {
         set_bootstrap_times(get_option_int(prefix + "BS_TIMES",
            get_bootstrap_times(),
            BOOTSTRAP_TIMES_HELP, true));
         set_bootstrap_fraction(get_option_real(prefix + "BS_FRACTION",
            get_bootstrap_fraction(),
            BOOTSTRAP_FRACTION_HELP, true));
      }

      if (perfEstimationMethod == holdOut || perfEstimationMethod == automatic)
      {
         set_holdout_times(get_option_int(prefix + "HO_TIMES",
            get_holdout_times(),
            HOLDOUT_TIMES_HELP, true));
         set_holdout_number(get_option_int(prefix + "HO_NUMBER",
            get_holdout_number(),
            HOLDOUT_NUMBER_HELP, true));
         if(get_holdout_number() == 0)
            set_holdout_percent(get_option_real_range(prefix + "HO_PERCENT",
               get_holdout_percent(),
               0, 1,
               HOLDOUT_PERCENT_HELP,
               true));
      }

      if (perfEstimationMethod == testSet)
      {
      // no more extra options for testSet
      }
   }

eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee 1*/


   // almost like the copy constructor, but doesn't copy perfData.  
   public void copy_options(PerfEstDispatch source)
{
   set_log_options(source.get_log_options());
   set_perf_estimator(source.get_perf_estimator());
   set_error_type(source.get_error_type());
   set_cv_weight_threshold(source.get_cv_weight_threshold());
   set_seed(source.get_seed());
   set_cv_folds(source.get_cv_folds());
   set_cv_times(source.get_cv_times());
   set_cv_fraction(source.get_cv_fraction());
   set_max_times(source.get_max_times());
   set_desired_std_dev(source.get_desired_std_dev());
   set_error_trim(source.get_error_trim());
   set_bootstrap_times(source.get_bootstrap_times());
   set_bootstrap_fraction(source.get_bootstrap_fraction());
   set_holdout_times(source.get_holdout_times());
   set_holdout_number(source.get_holdout_number());
   set_holdout_percent(source.get_holdout_percent());
}


   public double estimate_performance(BaseInducer inducer,
			  InstanceList trainList,
			  InstanceList testList){return estimate_performance(inducer,trainList,testList,null);}

/*   public double estimate_performance(BaseInducer baseInducer,
			  InstanceList trainList,
			  InstanceList testList,
			  PerfData providedPerfData)
   {
      // Create train and test InstanceLists as copies of passed-in InstanceLists.
      InstanceList newTrainInstList = (InstanceList)trainList.clone();

      //Added for estimate_performance(BaseInducer,InstanceList,PerfData) -JL
      InstanceList newTestInstList = null;
      if(testList != null)
         newTestInstList = (InstanceList)testList.clone();

//obs         InstanceList newTestInstList = (InstanceList)testList.clone();
   
      // Call the protected version.
      return estimate_performance(baseInducer, newTrainInstList,
			       newTestInstList, providedPerfData);
   }
*/

public double estimate_performance(BaseInducer baseInducer,
					   InstanceList trainInstList,
					   InstanceList testInstList,
					   PerfData providedPerfData)