dictionaryloader.java

A Java port of LDC s Champollion sentence aligner (http://champollion.sourceforge.net). Intended aud

package net.sourceforge.akerblad;
import info.jonclark.util.StringUtils;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;

public class DictionaryLoader {

    public static HashMap<String, ArrayList<String>> loadDict(String dictIn, HashSet<String> xstopHref)
	    throws IOException {

	System.err.println("Reading seed translation lexicon...");
	HashMap<String, ArrayList<String>> dictHref = new HashMap<String, ArrayList<String>>();

	// open D, "<$dict_fn" || die "$0: Couldn't open $dict_fn!\n";
	BufferedReader d = new BufferedReader(new FileReader(dictIn));

	String entry;
	while ((entry = d.readLine()) != null) {
	    String[] tokens = StringUtils.split(entry, " <> ", 2);

	    if (tokens.length == 2) {
		String source = tokens[0];
		String translation = tokens[1];

		source = source.toLowerCase();
		source = source.trim();
		translation = translation.trim();

		if (xstopHref.contains(source))
		    continue;

		// TODO: ?
		// #$translation =~ s/\W/\\$&/g;

		ArrayList<String> translations = dictHref.get(source);
		if(translations == null) {
		    translations = new ArrayList<String>();
		    translations.add(translation);
		    dictHref.put(source, translations);
		} else {
		    translations.add(translation);
		}
	    } else {
		System.err.println("invalid dictionary entry: " + entry);
	    }
	}

	d.close();
	System.err.println("done.");
	System.err.println("Number of entries: " + dictHref.size());

	return dictHref;
    }
}