evaluation.java

:<<数据挖掘--实用机器学习技术及java实现>>一书的配套源程序

/*
 *    This program is free software; you can redistribute it and/or modify
 *    it under the terms of the GNU General Public License as published by
 *    the Free Software Foundation; either version 2 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

/*
 *    Evaluation.java
 *    Copyright (C) 1999 Eibe Frank,Len Trigg
 *
 */

package weka.classifiers;

import java.util.*;
import java.io.*;
import weka.core.*;
import weka.estimators.*;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;

/**
 * Class for evaluating machine learning models. <p>
 *
 * Has been modified to run cross-validations in parallel for WEKA-PARALLEL.<p>
 *
 * ------------------------------------------------------------------- <p>
 *
 * General options when evaluating a learning scheme from the command-line: <p>
 *
 * -t filename <br>
 * Name of the file with the training data. (required) <p>
 *
 * -T filename <br>
 * Name of the file with the test data. If missing a cross-validation
 * is performed. <p>
 *
 * -c index <br>
 * Index of the class attribute (1, 2, ...; default: last). <p>
 *
 * -x number <br>
 * The number of folds for the cross-validation (default: 10). <p>
 *
 * -s seed <br>
 * Random number seed for the cross-validation (default: 1). <p>
 *
 * -m filename <br>
 * The name of a file containing a cost matrix. <p>
 *
 * -l filename <br>
 * Loads classifier from the given file. <p>
 *
 * -d filename <br>
 * Saves classifier built from the training data into the given file. <p>
 *
 * -v <br>
 * Outputs no statistics for the training data. <p>
 *
 * -o <br>
 * Outputs statistics only, not the classifier. <p>
 *
 * -i <br>
 * Outputs information-retrieval statistics per class. <p>
 *
 * -k <br>
 * Outputs information-theoretic statistics. <p>
 *
 * -p range <br>
 * Outputs predictions for test instances, along with the attributes in
 * the specified range (and nothing else). Use '-p 0' if no attributes are
 * desired. <p>
 *
 * -r <br>
 * Outputs cumulative margin distribution (and nothing else). <p>
 *
 * -g <br>
 * Only for classifiers that implement "Graphable." Outputs
 * the graph representation of the classifier (and nothing
 * else). <p>
 *
 * -a <br>
 * Causes the cross-validation portion of the program to occur in
 * parallel on multiple machines specified in ~/.weka-parallel.
 *
 * ------------------------------------------------------------------- <p>
 *
 * Example usage as the main of a classifier (called FunkyClassifier):
 * <code> <pre>
 * public static void main(String [] args) {
 *   try {
 *     Classifier scheme = new FunkyClassifier();
 *     System.out.println(Evaluation.evaluateModel(scheme, args));
 *   } catch (Exception e) {
 *     System.err.println(e.getMessage());
 *   }
 * }
 * </pre> </code>
 * <p>
 *
 * ------------------------------------------------------------------ <p>
 *
 * Example usage from within an application:
 * <code> <pre>
 * Instances trainInstances = ... instances got from somewhere
 * Instances testInstances = ... instances got from somewhere
 * Classifier scheme = ... scheme got from somewhere
 *
 * Evaluation evaluation = new Evaluation(trainInstances);
 * evaluation.evaluateModel(scheme, testInstances);
 * System.out.println(evaluation.toSummaryString());
 * </pre> </code>
 *
 *
 * @author   Eibe Frank ([e-mail])
 * @author   Len Trigg ([e-mail])
 * @author   Dave Musicant ([e-mail])
 * @author   Sebastian Celis ([e-mail])
 * @version  $Revision: 1.42 $
 */
public class Evaluation implements Summarizable, Serializable
{
    /** The number of classes. */
    private int m_NumClasses;

    /** The number of folds for a cross-validation. */
    private int m_NumFolds;

    /** The weight of all incorrectly classified instances. */
    private double m_Incorrect;

    /** The weight of all correctly classified instances. */
    private double m_Correct;

    /** The weight of all unclassified instances. */
    private double m_Unclassified;

    /*** The weight of all instances that had no class assigned to them. */
    private double m_MissingClass;

    /** The weight of all instances that had a class assigned to them. */
    private double m_WithClass;

    /** Array for storing the confusion matrix. */
    private double [][] m_ConfusionMatrix;

    /** The names of the classes. */
    private String [] m_ClassNames;

    /** Is the class nominal or numeric? */
    private boolean m_ClassIsNominal;

    /** The prior probabilities of the classes */
    private double [] m_ClassPriors;

    /** The sum of counts for priors */
    private double m_ClassPriorsSum;

    /** The cost matrix (if given). */
    private CostMatrix m_CostMatrix;

    /** The total cost of predictions (includes instance weights) */
    private double m_TotalCost;

    /** Sum of errors. */
    private double m_SumErr;

    /** Sum of absolute errors. */
    private double m_SumAbsErr;

    /** Sum of squared errors. */
    private double m_SumSqrErr;

    /** Sum of class values. */
    private double m_SumClass;

    /** Sum of squared class values. */
    private double m_SumSqrClass;

    /*** Sum of predicted values. */
    private double m_SumPredicted;

    /** Sum of squared predicted values. */
    private double m_SumSqrPredicted;

    /** Sum of predicted * class values. */
    private double m_SumClassPredicted;

    /** Sum of absolute errors of the prior */
    private double m_SumPriorAbsErr;

    /** Sum of absolute errors of the prior */
    private double m_SumPriorSqrErr;

    /** Total Kononenko & Bratko Information */
    private double m_SumKBInfo;

    /*** Resolution of the margin histogram */
    private static int k_MarginResolution = 500;

    /** Cumulative margin distribution */
    private double m_MarginCounts [];

    /** Number of non-missing class training instances seen */
    private int m_NumTrainClassVals;

    /** Array containing all numeric training class values seen */
    private double [] m_TrainClassVals;

    /** Array containing all numeric training class weights */
    private double [] m_TrainClassWeights;

    /** Numeric class error estimator for prior */
    private Estimator m_PriorErrorEstimator;

    /** Numeric class error estimator for scheme */
    private Estimator m_ErrorEstimator;

    /**
     * The minimum probablility accepted from an estimator to avoid
     * taking log(0) in Sf calculations.
     */
    private static final double MIN_SF_PROB = Double.MIN_VALUE;

    /** Total entropy of prior predictions */
    private double m_SumPriorEntropy;

    /** Total entropy of scheme predictions */
    private double m_SumSchemeEntropy;

    /**
     * Holds the names of all of the computers that helped contribute
     * to the cross-validation if parallelism occurred
     */
    private StringBuffer m_OtherComputers;

    /**
     * Initializes all the counters for the evaluation.
     *
     * @param data set of training instances, to get some header
     * information and prior class distribution information
     * @exception Exception if the class is not defined
     */
    public Evaluation(Instances data) throws Exception
    {
        this(data, null);
    }


    // The following are accessor methods, most of which were added
    // by Sebastian Celis for parallelization purposes.

    /**
     * Gets the number of classes
     *
     * @return the number of classes
     */
    public final int numClasses()
    {
        return m_NumClasses;
    }

    /**
     * Gets the number of folds to be done in the cross-validation step.
     *
     * @return the number of folds
     */
    public final int numFolds()
    {
        return m_NumFolds;
    }

    /**
     * Gets the number of instances incorrectly classified (that is, for
     * which an incorrect prediction was made). (Actually the sum of the
     * weights of these instances)
     *
     * @return the number of incorrectly classified instances
     */
    public final double incorrect()
    {
        return m_Incorrect;
    }

    /**
     * Gets the number of instances correctly classified (that is, for
     * which a correct prediction was made). (Actually the sum of the weights
     * of these instances)
     *
     * @return the number of correctly classified instances
     */
    public final double correct()
    {
        return m_Correct;
    }

    /**
     * Gets the number of instances not classified (that is, for
     * which no prediction was made by the classifier). (Actually the sum
     * of the weights of these instances)
     *
     * @return the number of unclassified instances
     */
    public final double unclassified()
    {
        return m_Unclassified;
    }

    /**
     * Gets the value of the missingClass variable
     *
     * @return the value of the missingClass variable
     */
    public final double missingClass()
    {
        return m_MissingClass;
    }

    /**
     * Gets the value of the withClass variable
     *
     * @return the value of the withClass variable
     */
    public final double withClass()
    {
        return m_WithClass;
    }

    /**
     * Returns a copy of the confusion matrix.
     *
     * @return a copy of the confusion matrix as a two-dimensional array
     */
    public double [][] confusionMatrix()
    {
        if(m_ConfusionMatrix == null)
            return null;

        double[][] newMatrix = new double[m_ConfusionMatrix.length][0];

        for (int i = 0; i < m_ConfusionMatrix.length; i++)
        {
            newMatrix[i] = new double[m_ConfusionMatrix[i].length];
            System.arraycopy(m_ConfusionMatrix[i], 0, newMatrix[i], 0,
                             m_ConfusionMatrix[i].length);
        }
        return newMatrix;
    }

    /**
     * Returns a copy of the classNames array.
     *
     * @return a copy of the classNames array
     */
    public String [] classNames()
    {
        String[] newNames = new String[m_ClassNames.length];

        for(int i = 0; i < m_ClassNames.length; i++)
        {
            newNames[i] = new String(m_ClassNames[i]);
        }
        return newNames;
    }

    /**
     * Tells whether or not the class is nominal.
     *
     * @return a boolean telling whether or not the class is nominal.
    public final boolean classIsNominal()
    {
        return m_ClassIsNominal;
    }

    /**
     * Gets a copy of the classPriors array.
     *
     * @return a copy of the classPriors array
     */
    public double [] classPriors()
    {
        double [] newPriors = new double[m_ClassPriors.length];

        for(int i = 0; i < m_ClassPriors.length; i++)
        {
            newPriors[i] = m_ClassPriors[i];
        }
        return newPriors;
    }

    /**
     * Gets the value of the classPriorsSum variable
     *
     * @return the value of the classPriorsSum variable
     */
    public final double classPriorsSum()
    {
        return m_ClassPriorsSum;
    }

    /**
     * Gets a copy of the cost matrix.
     *
     * @return a copy of the cost matrix
     */