rawconverter.java

自己写的search engine, 有 boolean search, fuzzy search

package searchingEngine.utilites.dataConverter;

import java.io.*;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;

import searchingEngine.dataPreprocessing.invertedFile.DocNode;
import searchingEngine.dataPreprocessing.invertedFile.InvertedFile;
import searchingEngine.dataPreprocessing.rawData.FileNode;
import searchingEngine.dataPreprocessing.rawData.PostNode;

public class RawConverter {
	public static final String FILETXT_PATH = "J:/WorkFolder/0506sem2/comp433/project/assign/file.txt";
	/**
	 * @param args
	 */
	public static  LinkedList<FileNode> loadFileList(String input,int offset,int len) throws IOException{
		FileNode tempNode;
		LinkedList<FileNode> tempList = new LinkedList<FileNode>();
		BufferedReader br = new BufferedReader(new FileReader(input));
		String fileLine;
		String attributes[];
		int count=0;
		long startTime;

		System.out.println("loadFileList - Start time: " + (startTime= System.nanoTime()) );
		skipLines(br,offset);
		while ((fileLine=br.readLine())!=null && (fileLine.trim().length()!=0) && count<len) {
			attributes = fileLine.split(" ");
			tempNode = new FileNode(Integer.parseInt(attributes[0]),Integer.parseInt(attributes[1]),attributes[2],attributes[3],attributes[4]);
			tempList.add(tempNode);
			count++;
			if (count%50000 == 0) System.out.println("count:"+count);
		}
		System.out.print(count + " items are read, time used: ");
		System.out.println(System.nanoTime() - startTime + " ");
		br.close();
		return tempList;
	}
	
	public static String[] getFileNameList(String input,int offset,int len) throws IOException {
		String temp[] = new String[InvertedFile.TOTAL_NUMBER_OF_FILES];
		LinkedList<FileNode> tempFileList = loadFileList(input, offset,len);
		for (int i=0;i<tempFileList.size();i++) {
				temp[i] = tempFileList.get(i).docid;
		}
		return temp;
	}
	
	public static FileNode loadFileNodeAt(String input,int offset)throws IOException{
		BufferedReader br = new BufferedReader(new FileReader(input));
		skipLines(br,offset);
		String fileLine = br.readLine();		
		br.close();
		if (fileLine!=null) {
			String attributes[] = fileLine.split(" ");
			return new FileNode(Integer.parseInt(attributes[0]),Integer.parseInt(attributes[1]),attributes[2],attributes[3],attributes[4]);
		}
		return null;
		
	}
	public static LinkedList<PostNode> loadPostList(String input,int offset,int len) throws IOException{
		PostNode tempNode;
		LinkedList<PostNode> tempList = new LinkedList<PostNode>();
		BufferedReader br = new BufferedReader(new FileReader(input));
		String fileLine;
		String attributes[];
		int id= 0;
		long startTime;
		System.out.println("loadPostList - Start time: " + (startTime= System.nanoTime()) );
		skipLines(br,offset);
		while ((fileLine=br.readLine())!=null && (fileLine.trim().length()!=0)&& id<len) {
			attributes = fileLine.split(" ");
			tempNode = new PostNode(id+offset,attributes[0],Integer.parseInt(attributes[1]),Integer.parseInt(attributes[2]));
			tempList.add(tempNode);
			id++;
			if (id%50000 == 0) System.out.println("count:"+id);
		}
		System.out.print(id + " items are read, time used: ");
		System.out.println(System.nanoTime() - startTime + " ");
		br.close();
		return  tempList;		
	}
	public static LinkedList<PostNode> loadSortedPostList(String input,int offset,int len) throws IOException{
		PostNode tempNode;
		LinkedList<PostNode> tempList = new LinkedList<PostNode>();
		BufferedReader br = new BufferedReader(new FileReader(input));
		String fileLine;
		String attributes[];
		int count= 0;
		long startTime;
		System.out.println("loadSortedPostList - Start time: " + (startTime= System.nanoTime()) );
		skipLines(br,offset);
		while ((fileLine=br.readLine())!=null && (fileLine.trim().length()!=0)&& count<len) {
			attributes = fileLine.split(" ");
			tempNode = new PostNode(Integer.parseInt(attributes[0]),attributes[1],Integer.parseInt(attributes[2]),Integer.parseInt(attributes[3]));
			tempList.add(tempNode);
			count++;
			if (count%50000 == 0) System.out.println("count:"+count);
		}
		System.out.print(count + " items are read, time used: ");
		System.out.println(System.nanoTime() - startTime + " ");
		br.close();
		return  tempList;		
	}
	
	public static void saveListAsText(List input,String output) throws IOException{
		long startTime;
		System.out.println("saveListAsText - Start time: " + (startTime= System.nanoTime()) );
		if (input.size()!=0) {
			FileWriter fw = new FileWriter(output);
		BufferedWriter bw =  new BufferedWriter(fw);
		for (int i=0;i<input.size();i++){
				bw.write(input.get(i).toString());
				bw.newLine();
				if (i%50000 == 0) System.out.println("ln" + i);
		}
		System.out.print(" Text file saved, time used: ");
		System.out.println(System.nanoTime() - startTime + " ");
		
		bw.close();
		fw.close();
		fw= null;
		bw = null;
		System.gc();
		} else System.out.println("EMPTY LIST!");
	}
	
	public static void readPartOfFile(String input,String output, int max) throws IOException{
		BufferedReader br = new BufferedReader(new FileReader(input));
		BufferedWriter wr = new BufferedWriter(new FileWriter(output));
		String fileLine;
		int count= 0;
		while (count<max) {
			fileLine=br.readLine();
			wr.write(fileLine,0,fileLine.length());
			wr.newLine();
			count++;
		}
		br.close();
		wr.close();
		System.out.println(count + " lines are transfered");
	}
	
	public static void saveObject(Object input,String output) throws IOException{
		long startTime;
		System.out.println("saveObject - Start time: " + (startTime= System.nanoTime()) );
		FileOutputStream fos = new FileOutputStream(output);
		ObjectOutputStream oos =  new ObjectOutputStream(fos);
		oos.writeObject(input);
		System.out.print(" Obj are saved, time used: ");
		System.out.println(System.nanoTime() - startTime + " ");
		fos.close();
		oos.close();
		fos = null;
		oos = null;
		System.gc();
	}
	
	public static void skipLines(BufferedReader br,int offset) throws IOException{
		int dumpCount=0;
		while ((dumpCount!=offset) && (br.readLine())!=null) {dumpCount++;}
	}
	
	public static Object loadObject(String input) throws IOException{
		FileInputStream fis = new FileInputStream(input);
		ObjectInputStream ois =  new ObjectInputStream(fis);
		Object output = null;
		long startTime = System.nanoTime();
		//System.out.println("loadObject - Start time: " + startTime );
		try {
			output=ois.readObject();
		} catch (ClassNotFoundException cnfe){
			System.err.println(cnfe+"-RawConverter.hava line88 ");
		}
		//System.out.println(" time used: " + (System.nanoTime() - startTime));
		System.out.print(System.nanoTime() - startTime + " ");
		return output;
	}
	//-------------------------------------------------------------------------------------------
	
	private final Class listType;
	private final Class nodeType;
	//private List theList;
	private BufferedReader br = null;
	private BufferedWriter bw = null;
	public final String POSTNODE_TYPE = "PostNode",FILENODE_TYPE = "FileNode";
	
	public RawConverter(Class listType,Class nodeType) throws InstantiationException, IllegalAccessException, ClassNotFoundException {

		this.listType = listType;
		this.nodeType = nodeType;
	}
	
	public void openLoader(String inputFile) throws FileNotFoundException{
		br = new BufferedReader(new FileReader(inputFile));
	}
	
	public Object loadNext() throws IOException {
		String buf = br.readLine();
		if (buf==null) {
			closeLoader();
		} else {
			String temp[];
			if (nodeType.getSimpleName().intern() == POSTNODE_TYPE.intern()) {
				temp = buf.split(" ");
				return new PostNode(Integer.parseInt(temp[0]),temp[1],Integer.parseInt(temp[2]),Integer.parseInt(temp[3]));
			} else if (nodeType.getSimpleName().intern() == FILENODE_TYPE.intern()) {
				temp = buf.split(" ");
				return new FileNode(Integer.parseInt(temp[0]),Integer.parseInt(temp[1]),temp[2],temp[3],temp[4]);
			}
		}
		return null;
	}
	
	public void closeLoader(){
		try {
			if (br!=null) br.close();
		} catch (IOException ingore) {}
	}
	
	public void openSaver(String outputFile,boolean append) throws IOException{
		closeSaver();
		bw = new BufferedWriter(new FileWriter(outputFile,append));
	}

	public void saveNext(Object node) throws IOException{
		bw.append(node.toString());
		bw.newLine();
	}

	public void closeSaver() {
		try {
			if (bw!=null) bw.close();
		} catch (IOException ingore) {}
	}
	
	public void closeAll() {
		closeLoader();
		closeSaver();
	}
	
	public void skipLines(int offset)throws IOException{
		this.skipLines(br,offset);
	}
	
	public static void main(String[] args) throws Exception {
		// TODO Auto-generated method stub
		RawConverter raw = new RawConverter(LinkedList.class ,PostNode.class);
		raw.openSaver("linkedlist.txt",true);
		raw.saveNext(new PostNode(3245,"test",3423432,4232432));
		raw.closeSaver();
	}

}