stemmer.java

一个很不错的词频统计程序,目前只支持英文,中文的本人正在修改中.改好后上传给大家分享

            break;
        case 'l':
            if (ends("bli")) {
                r("ble");
                break;
            }
            if (ends("alli")) {
                r("al");
                break;
            }
            if (ends("entli")) {
                r("ent");
                break;
            }
            if (ends("eli")) {
                r("e");
                break;
            }
            if (ends("ousli")) {
                r("ous");
                break;
            }
            break;
        case 'o':
            if (ends("ization")) {
                r("ize");
                break;
            }
            if (ends("ation")) {
                r("ate");
                break;
            }
            if (ends("ator")) {
                r("ate");
                break;
            }
            break;
        case 's':
            if (ends("alism")) {
                r("al");
                break;
            }
            if (ends("iveness")) {
                r("ive");
                break;
            }
            if (ends("fulness")) {
                r("ful");
                break;
            }
            if (ends("ousness")) {
                r("ous");
                break;
            }
            break;
        case 't':
            if (ends("aliti")) {
                r("al");
                break;
            }
            if (ends("iviti")) {
                r("ive");
                break;
            }
            if (ends("biliti")) {
                r("ble");
                break;
            }
            break;
        case 'g':
            if (ends("logi")) {
                r("log");
                break;
            }
        }
    }

    /* step4() deals with -ic-, -full, -ness etc. similar strategy to step3. */

    private final void step4() {
        switch (b[k]) {
        case 'e':
            if (ends("icate")) {
                r("ic");
                break;
            }
            if (ends("ative")) {
                r("");
                break;
            }
            if (ends("alize")) {
                r("al");
                break;
            }
            break;
        case 'i':
            if (ends("iciti")) {
                r("ic");
                break;
            }
            break;
        case 'l':
            if (ends("ical")) {
                r("ic");
                break;
            }
            if (ends("ful")) {
                r("");
                break;
            }
            break;
        case 's':
            if (ends("ness")) {
                r("");
                break;
            }
            break;
        }
    }

    /* step5() takes off -ant, -ence etc., in context <c>vcvc<v>. */

    private final void step5() {
        if (k == 0)
            return; /* for Bug 1 */
        switch (b[k - 1]) {
        case 'a':
            if (ends("al"))
                break;
            return;
        case 'c':
            if (ends("ance"))
                break;
            if (ends("ence"))
                break;
            return;
        case 'e':
            if (ends("er"))
                break;
            return;
        case 'i':
            if (ends("ic"))
                break;
            return;
        case 'l':
            if (ends("able"))
                break;
            if (ends("ible"))
                break;
            return;
        case 'n':
            if (ends("ant"))
                break;
            if (ends("ement"))
                break;
            if (ends("ment"))
                break;
            /* element etc. not stripped before the m */
            if (ends("ent"))
                break;
            return;
        case 'o':
            if (ends("ion") && j >= 0 && (b[j] == 's' || b[j] == 't'))
                break;
            /* j >= 0 fixes Bug 2 */
            if (ends("ou"))
                break;
            return;
        /* takes care of -ous */
        case 's':
            if (ends("ism"))
                break;
            return;
        case 't':
            if (ends("ate"))
                break;
            if (ends("iti"))
                break;
            return;
        case 'u':
            if (ends("ous"))
                break;
            return;
        case 'v':
            if (ends("ive"))
                break;
            return;
        case 'z':
            if (ends("ize"))
                break;
            return;
        default:
            return;
        }
        if (m() > 1)
            k = j;
    }

    /* step6() removes a final -e if m() > 1. */

    private final void step6() {
        j = k;
        if (b[k] == 'e') {
            int a = m();
            if (a > 1 || a == 1 && !cvc(k - 1))
                k--;
        }
        if (b[k] == 'l' && doublec(k) && m() > 1)
            k--;
    }

    /**
     * Stem the word placed into the Stemmer buffer through calls to add().
     * Returns true if the stemming process resulted in a word different from
     * the input. You can retrieve the result with
     * getResultLength()/getResultBuffer() or toString().
     */
    public void stem() {
        k = i - 1;
        if (k > 1) {
            step1();
            step2();
            step3();
            step4();
            step5();
            step6();
        }
        i_end = k + 1;
        i = 0;
    }

    /**
     * Test program for demonstrating the Stemmer. It reads text from a a list
     * of files, stems each word, and writes the result to standard output. Note
     * that the word stemmed is expected to be in lower case: forcing lower case
     * must be done outside the Stemmer class. Usage: Stemmer file-name
     * file-name ...
     */
    public static void main(String[] args) {
        char[] w = new char[501];
        Stemmer s = new Stemmer();
        for (int i = 0; i < args.length; i++)
            try {
                FileInputStream in = new FileInputStream(args[i]);

                try {
                    while (true)

                    {
                        int ch = in.read();
                        if (Character.isLetter((char) ch)) {
                            int j = 0;
                            while (true) {
                                ch = Character.toLowerCase((char) ch);
                                w[j] = (char) ch;
                                if (j < 500)
                                    j++;
                                ch = in.read();
                                if (!Character.isLetter((char) ch)) {
                                    /* to test add(char ch) */
                                    for (int c = 0; c < j; c++)
                                        s.add(w[c]);

                                    /* or, to test add(char[] w, int j) */
                                    /* s.add(w, j); */

                                    s.stem();
                                    {
                                        String u;

                                        /* and now, to test toString() : */
                                        u = s.toString();

                                        /*
                                         * to test getResultBuffer(),
                                         * getResultLength() :
                                         */
                                        /*
                                         * u = new String(s.getResultBuffer(),
                                         * 0, s.getResultLength());
                                         */

                                        System.out.print(u);
                                    }
                                    break;
                                }
                            }
                        }
                        if (ch < 0)
                            break;
                        System.out.print((char) ch);
                    }
                } catch (IOException e) {
                    System.out.println("error reading " + args[i]);
                    break;
                }
            } catch (FileNotFoundException e) {
                System.out.println("file " + args[i] + " not found");
                break;
            }
    }
}