knn.java

中文自动分类。使用spider抓取网络信息

package text_category;

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import edu.udo.cs.wvtool.main.WVTWordVector;

public class KNN {	
    
    private int k = 15;
    
    private double[] ClassSim = null;
    
    private static Map indexmap = null;
    
    static {
    	if (indexmap == null)
    	{
    		indexmap = new HashMap();
    		indexmap.put(0, "姹借溅");
    		indexmap.put(1, "鏁欒偛");
    		indexmap.put(2, "濞变箰");
    		indexmap.put(3, "璐㈢粡");
    		indexmap.put(4, "鎴夸骇");
    		indexmap.put(5, "鍐涗簨");
    		indexmap.put(6, "濂ヨ繍");
    		indexmap.put(7, "鏃舵斂");
    		indexmap.put(8, "浣撹偛");
    		indexmap.put(9, "绉戞妧");
    	}
    }
    
    public KNN()
    {
    	
    }
	
	public List LazyLearning(WVTWordVector v, WVTWordVector[] vectors, int numClasses)	
	{
		if (v == null || vectors == null)
			return null;
		
		System.out.println("number of documents : " + vectors.length);
		System.out.println("number of classes: " + numClasses);
		ClassSim = new double[numClasses];
		for (int i = 0; i < numClasses; i++)
		{
			ClassSim[i] = 0;
		}
		
		k = (k < vectors.length)? k : vectors.length;
		double[] Sim = new double[vectors.length];
		for (int i = 0; i < Sim.length; i++)
		{
			Sim[i] = 0;
			Map map1 = v.getWordMap();
			Map map2 = vectors[i].getWordMap();
			for (Iterator it = map1.keySet().iterator(); it.hasNext();)
			{
				String word1 = (String)it.next();
				if (map2.containsKey(word1))
				{
					double value1 = Double.valueOf(map1.get(word1).toString());
					double value2 = Double.valueOf(map2.get(word1).toString());
					Sim[i] += (value1 * value2);
				}
			}
		}
		
		for (int i = 0; i < k; i++)
		{
			for (int j = i + 1; j < Sim.length; j++)
			{
				if (Sim[j] > Sim[i])
				{
					double dtemp = Sim[i];
					Sim[i] = Sim[j];
					Sim[j] = dtemp;
					
					WVTWordVector wv = vectors[i];
					vectors[i] = vectors[j];
					vectors[j] = wv;
				}
			}
		}
		
		double TotalSim = 0;
		for (int i = 0; i < k; i++)
		{
			WVTWordVector wv = vectors[i];
			int numClass = wv.getDocumentInfo().getClassValue();
			ClassSim[numClass] += Sim[i];
			TotalSim += Sim[i];
		}
		
		// output the first 3 class
		int[] index = new int[ClassSim.length];
		for (int i = 0; i < ClassSim.length; i++)
			index[i] = i;
		for (int i = 0; i < 3; i++)
		{			
			for (int j = i + 1; j < ClassSim.length; j++)
			{
				if (ClassSim[j] > ClassSim[i])
				{
					double dtemp = ClassSim[i];
					ClassSim[i] = ClassSim[j];
					ClassSim[j] = dtemp;
					
					int itemp = index[i];
					index[i] = index[j];
					index[j] = itemp;
				}
			}
		}
		
		List result = new ArrayList();
		for (int i = 0; i < 3; i++)
		{
			if (ClassSim[i] > 0)
			{
				CategoryResult cr = new CategoryResult(indexmap.get(index[i]).toString(), ClassSim[i] / TotalSim);
				result.add(cr);
			}
		}
		
		
		for (int i = 0; i < index.length; i++)
		{
			System.out.println(index[i] + ": " + ClassSim[i]);
		}			
		
		for (int i = 0; i < k; i++)
		{
			WVTWordVector wv = vectors[i];
			String id;
	        /*int cutIndex = wv.getDocumentInfo().getSourceName().lastIndexOf(File.separator);

	        if (cutIndex > 0)
	            id = wv.getDocumentInfo().getSourceName().substring(cutIndex + 1);
	        else*/
	            id = wv.getDocumentInfo().getSourceName();
	        
	        System.out.println(id + ": " + Sim[i]);
		}
		
		return result;
	}
		
}

class CategoryResult
{
	private String CategoryName;
	private double similarity;
	
	public CategoryResult(String categoryname, double similarity)
	{
		this.CategoryName = categoryname;
		this.similarity = similarity;
	}

	public String getCategoryName() {
		return CategoryName;
	}

	public void setCategoryName(String categoryName) {
		CategoryName = categoryName;
	}

	public double getSimilarity() {
		return similarity;
	}

	public void setSimilarity(double similarity) {
		this.similarity = similarity;
	}
}