ttest.cpp

gibbs

/*
 * ttestwcpp
 *
 * Compare a WinMine-generated Bayesian network to an NBE model by 
 * computing the log likelihoods of the observations in a user-specified
 * test set.  Performs a paired t-test for significance testing.
 *
 * Author: Daniel Lowd <[e-mail]>
 */

#include "BayesNet.h"
#include "AbsVarAbstractionSet.h"

#include <math.h>
#include <fstream>
using namespace std;

double getLogLikelihood(const BayesNet* model, const VarSet& t)
{
    double ll = 0.0;
    int numvars = t.getNumVars();
    double* allvars = new double[numvars];

    for (int i = 0; i < numvars; i++) {
        allvars[i] = t[i];
    }

    for (int i = 0; i < numvars; i++) {
        ll += model->getLogProb(i, allvars);
    }

    delete[] allvars;
    return ll;
}


int main(int argc, char* argv[])
{
    // Check arguments
    if (argc < 4) {
        cerr << "Usage: ./ttest <nbe model> <winmine model> <testdata>\n";
        return 1;
    }

    int nextArg = 1;
    bool reportfull = false;

    if (!strcmp(argv[nextArg], "-v")) {
        reportfull = true;
        nextArg++;
    }

    // Load NBE model
    ifstream modelin(argv[nextArg]);
    if (!modelin) {
        cerr << "Error: could not open NBE model file \"" 
            << argv[nextArg] << "\".\n";
        return 1;
    }
    nextArg++;

    AbsVarAbstractionSet nbe;
    modelin >> nbe;


    // Load WinMine model
    FILE* inFile = fopen(argv[nextArg], "r");

    if (!inFile) {
        cerr << "ERROR: could not open WinMine model file \"" 
            << argv[nextArg] << "\"\n";
        exit(-1);
    }
    nextArg++;

    BayesNet* model = new BayesNet();
    loadModel(inFile, *model);

    // DEBUG
    //cout << model;

    // Open test file
    ifstream testin(argv[nextArg]);
    if (!testin) {
        cerr << "Error: could not open test file \"" << argv[nextArg] << "\".\n";
        return 1;
    }
    nextArg++;

    // Read in each observation and compute its log likelihood
    double loglikelihood = 0.0;
    double nbe_loglikelihood = 0.0;
    double wm_loglikelihood = 0.0;
    double sq_loglikelihood = 0.0;
    int observations = 0;
    VarSet t;
    while (testin >> t) {
        double l_nbe = nbe.getLogLikelihood(t);
        double l_wm = getLogLikelihood(model, t);
        nbe_loglikelihood += l_nbe;
        wm_loglikelihood  += l_wm;
        loglikelihood += (l_nbe - l_wm);
        sq_loglikelihood += (l_nbe - l_wm) * (l_nbe - l_wm);
        observations++;
        // DEBUG
        //cout << loglikelihood/observations << endl;
    }

    cout << "NBE:     " << nbe_loglikelihood/observations << endl;
    cout << "WinMine: " << wm_loglikelihood/observations << endl;

    // Output results (both log likelihood and stdev)
    cout << "log(likelihood) = " << loglikelihood/observations << endl;
    double sigma_sq = (sq_loglikelihood - (loglikelihood*loglikelihood)/observations)
          /(observations*observations);
    cout << "sigma = " << sqrt(sigma_sq) << endl;

    if (reportfull) {
        cout << "sum = " << loglikelihood << endl;
        cout << "sumsq = " << sq_loglikelihood << endl;
        cout << "n = " << observations << endl;
    }
    return 0;
}