lengthstopfiltertokenizertest.java

一个自然语言处理的Java开源工具包。LingPipe目前已有很丰富的功能

package com.aliasi.test.unit.tokenizer;

import com.aliasi.tokenizer.IndoEuropeanTokenizerFactory;
import com.aliasi.tokenizer.LengthStopFilterTokenizer;
import com.aliasi.tokenizer.Tokenizer;
import com.aliasi.tokenizer.TokenizerFactory;

import com.aliasi.test.unit.BaseTestCase;

public class LengthStopFilterTokenizerTest extends BaseTestCase {

    public void testFilter() {
        testTokenizer(0,"foo bar",new String[] { });
        testTokenizer(3,"foo bar",new String[] { "foo", "bar" });
        testTokenizer(2,"foo ba", new String[] { "ba" });
        testTokenizer(2,"ba foo", new String[] { "ba" });
        testTokenizer(2,"a bc def a gh", new String[] { "a", "bc", "a", "gh" });
    }

    void testTokenizer(int max, String in, String[] expectedTokens) {
        Tokenizer base = IndoEuropeanTokenizerFactory.FACTORY.tokenizer(in.toCharArray(),0,in.length());
        Tokenizer filtered = new LengthStopFilterTokenizer(base,max);
        String[] tokens = filtered.tokenize();
        assertEqualsArray(expectedTokens,tokens);
    }

}