statisticstest.java

一个自然语言处理的Java开源工具包。LingPipe目前已有很丰富的功能

        try {
            matrix = new double[][] { { 1, -2, 3}, {4, 5, 6} };
            Statistics.chiSquaredIndependence(matrix);
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }

        try {
            matrix = new double[][] { { 1, 2, 3}, {4, Double.NaN, 6} };
            Statistics.chiSquaredIndependence(matrix);
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }

        try {
            matrix = new double[][] { { 1, 2, 3}, {4, 5, Double.POSITIVE_INFINITY} };
            Statistics.chiSquaredIndependence(matrix);
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }

    }

    public void testNormalize() {
        try {
            Statistics.normalize(new double[] { -1 });
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }

        try {
            Statistics.normalize(new double[] { 0, 2, -1, 5 });
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }

        try {
            Statistics.normalize(new double[] { 0, 2, Double.NaN, 5 });
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }

        try {
            Statistics.normalize(new double[] { 0, 2, Double.POSITIVE_INFINITY, 5 });
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }

        try {
            Statistics.normalize(new double[] { 0, 0, 0 });
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }


        try {
            Statistics.normalize(new double[] { });
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }



        assertEquals(1.0,
                     com.aliasi.util.Math.sum(Statistics.normalize(new double[] { 0, 1, 2, 3 })),
                     0.0001);
        assertEquals(1.0,
                     Statistics.normalize(new double[] { 17 })[0],
                     0.0001);
    }

    public void testChiSquare() {
        double both = 3;
        double oneOnly = 1;
        double twoOnly = 2;
        double neither = 4; // total = 10
        double pOne = .40;
        double pTwo = .50;

        double eBoth = 10.0 * pOne * pTwo;
        double eOne = 10.0 * pOne * (1.0 - pTwo);
        double eTwo = 10.0 * (1.0 - pOne) * pTwo;
        double eNeither = 10.0 * (1.0 - pOne) * (1.0 - pTwo);

        double diffBoth = both - eBoth;
        double diffOne = oneOnly - eOne;
        double diffTwo = twoOnly - eTwo;
        double diffNeither = neither - eNeither;

        double eChiSquare
            = diffBoth * diffBoth / eBoth
            + diffOne * diffOne / eOne
            + diffTwo * diffTwo / eTwo
            + diffNeither * diffNeither / eNeither;

        assertEquals(eChiSquare,
                     Statistics
                     .chiSquaredIndependence(both,oneOnly,twoOnly,neither),
                     0.0005);

        // answer derived from http://math.hws.edu/javamath/ryan/ChiSquare.html
        assertEquals(1.66666,
                     Statistics
                     .chiSquaredIndependence(both,oneOnly,twoOnly,neither),
                     0.0005);

        try {
            Statistics.chiSquaredIndependence(-1,2,3,4);
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }

        try {
            Statistics.chiSquaredIndependence(1,-2,3,4);
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }

        try {
            Statistics.chiSquaredIndependence(1,2,-3,4);
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }

        try {
            Statistics.chiSquaredIndependence(1,2,3,-4);
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }

    }

    public void testMeanVarDev() {
        double[] xs = new double[0];
        assertTrue(Double.isNaN(Statistics.mean(xs)));
        assertTrue(Double.isNaN(Statistics.variance(xs)));
        assertTrue(Double.isNaN(Statistics.standardDeviation(xs)));

        xs = new double[] { 1.0 };
        assertEquals(1.0, Statistics.mean(xs), 0.0001);
        assertEquals(0.0, Statistics.variance(xs), 0.0001);
        assertEquals(0.0, Statistics.standardDeviation(xs), 0.0001);

        xs = new double[] { 1.0, 3.0 };
        assertEquals(2.0, Statistics.mean(xs), 0.0001);
        assertEquals(1.0, Statistics.variance(xs), 0.0001);
        assertEquals(1.0, Statistics.standardDeviation(xs), 0.0001);

        xs = new double[] { 1.0, 3.0, 1.0, 3.0 };
        assertEquals(2.0, Statistics.mean(xs), 0.0001);
        assertEquals(1.0, Statistics.variance(xs), 0.0001);
        assertEquals(1.0, Statistics.standardDeviation(xs), 0.0001);
    }

    public void testSampling() {
        Random baseRandom = new Random();

        for (int k = 0; k < 100; ++k) {
            int numTopics = baseRandom.nextInt(300) + 1;
            double[] probRatios = new double[numTopics];
            for (int i = 0; i < numTopics; ++i) {
                probRatios[i] = (i == 0) ? 0.0 : probRatios[i-1];
                if (baseRandom.nextDouble() > 0.1)
                    probRatios[i] += baseRandom.nextDouble() * 100.0;
            }

            for (int j = 0; j < 100; ++j) {
                long seed = baseRandom.nextLong();
                double x = new Random(seed).nextDouble() * probRatios[numTopics-1];
                int sample = Statistics.sample(probRatios,new Random(seed));
                assertTrue(x <= probRatios[sample]);
                assertTrue(sample == 0
                           || x > probRatios[sample-1]);
            }
        }


    }

    public void testDirichlet() {
        assertDirichlet(2,new double[] { 0.5, 0.5 }, 1.5);
        assertDirichlet(new double[] { 2.0, 2.0 },
                        new double[] { 0.5, 0.5 },
                        1.5);
        double log2Expected = com.aliasi.util.Math.log2Gamma(2 + 2)
            - 2 * com.aliasi.util.Math.log2Gamma(2)
            + com.aliasi.util.Math.log2(0.25)
            + com.aliasi.util.Math.log2(0.75);
        assertDirichlet(2,
                        new double[] { 0.25, 0.75 },
                        Math.pow(2.0,log2Expected));


        double log2Expected2 = com.aliasi.util.Math.log2Gamma(3 + 4 + 5)
            - com.aliasi.util.Math.log2Gamma(3)
            - com.aliasi.util.Math.log2Gamma(4)
            - com.aliasi.util.Math.log2Gamma(5)
            + com.aliasi.util.Math.log2(Math.pow(0.2,3-1))
            + com.aliasi.util.Math.log2(Math.pow(0.3,4-1))
            + com.aliasi.util.Math.log2(Math.pow(0.5,5-1));

        assertDirichlet(new double[] { 3, 4, 5 },
                        new double[] { 0.2, 0.3, 0.5 },
                        Math.pow(2.0, log2Expected2));





    }

    void assertDirichlet(double alpha, double[] xs, double expectedP) {
        double expectedLog2P = com.aliasi.util.Math.log2(expectedP);
        assertEquals(expectedLog2P,
                     Statistics.dirichletLog2Prob(alpha,xs),
                     0.0001);
    }

    void assertDirichlet(double[] alphas, double[] xs, double expectedP) {
        double expectedLog2P = com.aliasi.util.Math.log2(expectedP);
        assertEquals(expectedLog2P,
                     Statistics.dirichletLog2Prob(alphas,xs),
                     0.0001);
    }

    public void testDirichletExceptions() {
        assertDirichletFail(-1,new double[] { 0.5, 0.5 });
        assertDirichletFail(0.0,new double[] { 0.5, 0.5 });
        assertDirichletFail(Double.NaN,new double[] { 0.5, 0.5 });
        assertDirichletFail(Double.POSITIVE_INFINITY,new double[] { 0.5, 0.5 });

        assertDirichletFail(new double[] { 0.4, -1 }, new double[] { 0.25, 0.75 });
        assertDirichletFail(new double[] { 0.4, 0 }, new double[] { 0.25, 0.75 });
        assertDirichletFail(new double[] { Double.NaN, 0.4 }, new double[] { 0.25, 0.75 });
        assertDirichletFail(new double[] { 0.4, 0.4, Double.POSITIVE_INFINITY },
                            new double[] { 0.25, 0.5, 0.5 });

        assertDirichletFail(1, new double[] { -1, 0.5 });
        assertDirichletFail(new double[] { 1, 1 }, new double[] { -1, 0.5 });

        assertDirichletFail(1, new double[] { 0.25, 2 });
        assertDirichletFail(new double[] { 1, 1 }, new double[] { 0.5, 2 });

        assertDirichletFail(1, new double[] { 0.25, Double.NEGATIVE_INFINITY });
        assertDirichletFail(new double[] { 1, 1 }, new double[] { 0.5, Double.POSITIVE_INFINITY });
        assertDirichletFail(new double[] { 1, 1 }, new double[] { 0.5, Double.NaN });


        assertDirichletFail(new double[] { 1, 2, 3 }, new double[] { 0.5, 0.5 });
    }

    void assertDirichletFail(double alpha, double[] xs) {
        try {
            Statistics.dirichletLog2Prob(-1,new double[] { 0.5, 0.5 });
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }
    }

    void assertDirichletFail(double[] alphas, double[] xs) {
        try {
            Statistics.dirichletLog2Prob(alphas,xs);
            fail();
        } catch (IllegalArgumentException e) {
            succeed();
        }

    }

}