statisticstest.java

一个自然语言处理的Java开源工具包。LingPipe目前已有很丰富的功能

package com.aliasi.test.unit.stats;

import com.aliasi.stats.Statistics;

import com.aliasi.test.unit.BaseTestCase;

import java.util.HashSet;
import java.util.Random;
import java.util.Set;

public class StatisticsTest extends BaseTestCase {


    public void testDirDivergence() {
	Random random = new Random();
	for (int k = 0; k < 100; ++k) {
	    double[] xs = new double[random.nextInt(100)+2];
	    double[] ys = new double[xs.length];
	    for (int i = 0; i < xs.length; ++i)
		xs[i] = random.nextDouble() * 10.0;
	    for (int i = 0; i < xs.length; ++i)
		ys[i] = random.nextDouble() * 10.0;
	    assertEquals(0.0,Statistics.klDivergenceDirichlet(xs,xs), 0.0001);
	    assertTrue(0.0 <= Statistics.klDivergenceDirichlet(xs,ys));
	}
    }

    public void testDivergenceExceptions() {
	double[] p = new double[] { 0.5, 0.2, 0.3 };
	double[] q = new double[] { 0.2, 0.8 };
	double[] r = new double[] { 0.1, -0.2, 0.7 };
	double[] s = new double[] { 0.1, 1.2, 0.7 };
	double[] t = new double[] { 0.1, Double.POSITIVE_INFINITY, 0.7 };
	double[] u = new double[] { Double.NEGATIVE_INFINITY, 0.7, 0.3 };
	double[] v = new double[] { 0.1, 0.2, Double.NaN };
	assertFailDivergence(p,q);
	assertFailDivergence(p,r);
	assertFailDivergence(p,s);
	assertFailDivergence(p,t);
	assertFailDivergence(p,u);
	assertFailDivergence(p,v);
    }

    void assertFailDivergence(double[] p, double[] q) {
	try {
	    Statistics.klDivergence(p,q);
	    fail();
	} catch (IllegalArgumentException e) {
	    succeed();
	}
	try {
	    Statistics.klDivergence(q,p);
	    fail();
	} catch (IllegalArgumentException e) {
	    succeed();
	}

	try {
	    Statistics.symmetrizedKlDivergence(p,q);
	    fail();
	} catch (IllegalArgumentException e) {
	    succeed();
	}
	try {
	    Statistics.symmetrizedKlDivergence(q,p);
	    fail();
	} catch (IllegalArgumentException e) {
	    succeed();
	}

	try {
	    Statistics.jsDivergence(p,q);
	    fail();
	} catch (IllegalArgumentException e) {
	    succeed();
	}
	try {
	    Statistics.jsDivergence(q,p);
	    fail();
	} catch (IllegalArgumentException e) {
	    succeed();
	}
    }

    public void testEqualDivergences() {
	double[] p = new double[] { 0.1, 0.4, 0.5 };
	double[] q = new double[] { 0.1, 0.4, 0.5 };
	assertEquals(0.0,Statistics.klDivergence(p,q),0.0001);
	assertEquals(0.0,Statistics.symmetrizedKlDivergence(p,q),0.0001);
	assertEquals(0.0,Statistics.jsDivergence(p,q),0.0001);

	double[] r = new double[0];
	assertEquals(0.0,Statistics.klDivergence(r,r),0.0001);
	assertEquals(0.0,Statistics.symmetrizedKlDivergence(r,r),0.0001);
	assertEquals(0.0,Statistics.jsDivergence(r,r),0.0001);

	double[] s = new double[] { 1.0 };
	assertEquals(0.0,Statistics.klDivergence(s,s),0.0001);
	assertEquals(0.0,Statistics.symmetrizedKlDivergence(s,s),0.0001);
	assertEquals(0.0,Statistics.jsDivergence(s,s),0.0001);
    }

    public void testDivergences() {
	double[] p = new double[] { 0.4, 0.6 };
	double[] q = new double[] { 0.6, 0.4 };
	double expectedKl = 0.4 * com.aliasi.util.Math.log2(0.4/0.6)
	    + 0.6 * com.aliasi.util.Math.log2(0.6/0.4);
	assertEquals(expectedKl,Statistics.klDivergence(p,q),0.0001);
	
	double expectedSkl = expectedKl;
	assertEquals(expectedSkl,Statistics.symmetrizedKlDivergence(p,q),0.0001);
	assertEquals(expectedSkl,Statistics.symmetrizedKlDivergence(q,p),0.0001);

	double expectedJs = 0.4 * com.aliasi.util.Math.log2(0.4/0.5)
	    + 0.6 * com.aliasi.util.Math.log2(0.6/0.5);
	assertEquals(expectedJs,Statistics.jsDivergence(p,q),0.0001);
	assertEquals(expectedJs,Statistics.jsDivergence(q,p),0.0001);
    }


    public void testPermutation() {
        int[] xs = Statistics.permutation(0);
        assertEquals(0,xs.length);

        xs = Statistics.permutation(1);
        assertEquals(1,xs.length);
        assertEquals(0,xs[0]);

        xs = Statistics.permutation(2);
        assertEquals(2,xs.length);
        assertTrue(xs[0] == 0 && xs[1] == 1
                   || xs[0] == 1 && xs[1] == 0);

        xs = Statistics.permutation(100);
        Set<Integer> xSet = new HashSet<Integer>(200);
        for (int i = 0; i < xs.length; ++i) {
            assertTrue(0 <= xs[i] && xs[i] < 100);
            xSet.add(new Integer(xs[i]));
        }
        assertEquals(100,xSet.size());
    }

    public void testRegressionExceptions() {
        double[] xs = { 1 };
        double[] ys = { 2 };
        try {
            Statistics.linearRegression(xs,ys);
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }
        try {
            Statistics.logisticRegression(xs,ys,2.0);
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }

        double[] xs2 = { 1, 2 };
        double[] ys3 = { 1, 2, 3 };
        try {
            Statistics.linearRegression(xs2,ys3);
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }
        try {
            Statistics.logisticRegression(xs2,ys3,1.0);
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }

        double[] xs4 = { 1, 1, 1, 1 };
        double[] ys4 = { 2, 2, 2, 2 };
        try {
            Statistics.linearRegression(xs4,ys4);
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }
        try {
            Statistics.logisticRegression(xs4,ys4,2.0);
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }

        double[] xs3 = { 5, 9, 7 };
        try {
            Statistics.logisticRegression(xs3,ys3,-1);
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }
    }

    // Case Study 11.25, p. 577 Larsen & Marx. 2001. An Introduction
    // to Mathematical Statistics and Its Applications (3rd
    // Ed). Prentice-Hall.
    //
    public void testLogisticRegression() {
        double[] xs = { 480, 690, 900, 1100, 1320, 1530 };
        double[] ys = { 0.3, 4.6, 15.6, 33.4, 44.4, 45.7 };
        double[] betas = Statistics.logisticRegression(xs,ys,48);
        assertEquals(7.91, betas[0], 0.1);
        assertEquals(-0.0076, betas[1], .0001);
    }

    public void testRegression() {
        double[] xs = { 1, 2 };
        double[] ys = { 3, 4 };
        double[] betas = Statistics.linearRegression(xs,ys);
        assertEquals(betas[0],2.0,0.001);
        assertEquals(betas[1],1.0,0.001);

    }

    // from Larsen & Marx, case study 11.2.1, p. 561
    public void testRegression2() {
        double[] xs = { 2.745, 2.700, 2.690, 2.680, 2.675, 2.670,
                        2.665, 2.660, 2.655, 2.655, 2.650, 2.650,
                        2.645, 2.635, 2.630, 2.625, 2.625, 2.620,
                        2.615, 2.615, 2.615, 2.610, 2.590, 2.590,
                        2.565 };
        double[] ys = { 2.080, 2.045, 2.050, 2.005, 2.035, 2.035,
                        2.020, 2.005, 2.010, 2.000, 2.000, 2.005,
                        2.015, 1.990, 1.990, 1.995, 1.985, 1.970,
                        1.985, 1.990, 1.995, 1.990, 1.975, 1.995,
                        1.955 };
        double[] betas = Statistics.linearRegression(xs,ys);
        assertEquals(betas[0],0.308,0.01);
        assertEquals(betas[1],0.642,0.01);
    }

    public void testCorrelation() {
        try {
            Statistics.correlation(new double[] { 1, 2 }, new double[] { 3, 4, 5 });
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }

        double[] xs = new double[] { 61, 61, 62.5, 63, 66, 70, 73, 75.5 };
        double[] ys = new double[] { 61, 63, 65, 63, 67, 72, 74, 75.5 };
        assertEquals(0.983798,Statistics.correlation(xs,ys),0.0001);

        double[] zs1 = { 1, 2, 3 };
        double[] zs2 = { 2, 4, 6 };
        double[] zs3 = { -1, -2, -3};
        assertEquals(1.0,Statistics.correlation(zs1,zs2),0.0001);
        assertEquals(1.0,Statistics.correlation(zs1,zs3),0.0001);
        assertEquals(1.0,Statistics.correlation(zs2,zs3),0.0001);
    }

    public void testChiSquareMatrix() {
        // Larsen & Marx. p. 551
        double[][] matrix
            = new double[][]
            { { 70, 65 },
              { 39, 28 },
              { 14,  3 },
              { 13,  2 } };
        assertEquals(11.3,
                     Statistics.chiSquaredIndependence(matrix),
                     0.1);  // textbook rounding

        // Larsen & Marx. p. 552
        matrix
            = new double[][]
            { { 24, 8, 13 },
              { 8, 13, 11 },
              { 10, 9, 64 } }; // Siskel & Ebert ratings down/sideways/up
        assertEquals(45.37,
                     Statistics.chiSquaredIndependence(matrix),
                     0.1); // textbook rounding

        try {
            matrix = new double[][] { { 1, 2}, {3, 5, 6} };
            Statistics.chiSquaredIndependence(matrix);
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }