umlstokenlist.java

dragontoolkit用于机器学习

package dragon.nlp.ontology.umls;

import dragon.nlp.Token;
import dragon.util.SortedArray;
import dragon.util.FileUtil;
import java.util.*;
import java.io.*;

/**
 * <p>List for storing UMLS tokens</p>
 * <p> </p>
 * <p>Copyright: Copyright (c) 2005</p>
 * <p>Company: IST, Drexel University</p>
 * @author Davis Zhou
 * @version 1.0
 */

public class UmlsTokenList extends SortedArray{
	private static final long serialVersionUID = 1L;

	public UmlsTokenList(){

    }

    public UmlsTokenList(String tokenFile) {
        loadTokenList(tokenFile);
    }

    public int addToken(Token token){
        token.setIndex(size());
        if (this.add(token)) {
            token.setFrequency(1);
            return (token.getIndex());
        }
        else {
            token = (Token) get(insertedPos());
            token.addFrequency(1);
            return (token.getIndex());
        }
    }

    public Token tokenAt(int index){
        return (Token)get(index);
    }

    public Token lookup(String token){
        int pos;

        pos=binarySearch(new Token(token));
        if(pos<0)
            return null;
        else
            return (Token)get(pos);
    }

    public Token lookup(Token token){
        int pos;

        pos=binarySearch(token);
        if(pos<0)
            return null;
        else
            return (Token)get(pos);
    }

    private void loadTokenList(String tokenFilename) {
        int len, freq;
        String line;
        ArrayList tokenList;
        BufferedReader br;
        Token token;

        try {
            System.out.println(new java.util.Date() + " Loading Token List...");
            br = FileUtil.getTextReader(tokenFilename);
            line = br.readLine();
            len = Integer.parseInt(line);
            tokenList = new ArrayList(len);

            for (int i = 0; i < len; i++) {
                line = br.readLine();
                String[] strArr = line.split("\t");
                token = new Token(strArr[1]);
                token.setIndex(Integer.parseInt(strArr[0]));
                freq = Integer.parseInt(strArr[2]);
                token.setFrequency(freq);
                token.setWeight(1.0/freq);
                tokenList.add(token);
            }
            br.close();
            Collections.sort(tokenList);
            this.addAll(tokenList);
        }
        catch (Exception ex) {
            ex.printStackTrace();
        }
    }

    public void saveTo(String filename) {
        BufferedWriter bw;
        Token t;
        int i;

        try {
            System.out.println(new java.util.Date() + " Saving Token List...");
            bw = FileUtil.getTextWriter(filename);
            bw.write(size() + "");
            bw.write("\n");
            for (i = 0; i < size(); i++) {
                t = (Token) get(i);
                bw.write(t.getIndex() + "\t" + t.getValue() + "\t" + t.getFrequency() + "\n");
                bw.flush();
            }
            bw.close();
        }
        catch (Exception ex) {
            ex.printStackTrace();
        }
    }

}