indexer.java

基于lucene的 全文检索程序。可以对office等文件pdf文件进行检索

package org.tatan.framework;

import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.analysis.cjk.CJKAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;

import java.util.Date;


public class Indexer {

  public static void main(String[] args) throws Exception {
    
    File indexDir = new File("d:/testdoc/index");
    File dataDir = new File("c:/s");

    long start = new Date().getTime();
    int numIndexed = index(indexDir, dataDir);
    long end = new Date().getTime();

    System.out.println("Indexing " + numIndexed + " files took "
      + (end - start) + " milliseconds");
  }

  public static int index(File indexDir, File dataDir)
    throws Exception {

    if (!dataDir.exists() || !dataDir.isDirectory()) {
      throw new IOException(dataDir
        + " does not exist or is not a directory");
    }

     IndexWriter writer = new IndexWriter(indexDir,new CJKAnalyzer(), true);
    writer.setUseCompoundFile(false);

    indexDirectory(writer, dataDir);

    int numIndexed = writer.docCount();
    writer.optimize();
    writer.close();
    return numIndexed;
  }

  private static void indexDirectory(IndexWriter writer, File dir)
    throws Exception {

    File[] files = dir.listFiles();

    for (int i = 0; i < files.length; i++) {
      File f = files[i];
      if (f.isDirectory()) {
        indexDirectory(writer, f);  // recurse
      } else if (f.getName().endsWith(".doc")) {
        indexFile(writer, f);
      }
    }
  }

  private static void indexFile(IndexWriter writer, File f)
    throws Exception {

    if (f.isHidden() || !f.exists() || !f.canRead()) {
      return;
    }

    System.out.println("Indexing " + f.getCanonicalPath());

    Document doc = new Document();
    POIWordDocHandler handler = new POIWordDocHandler();
    Field FieldBody = new Field("body", handler.getDocument(new FileInputStream(f)), Field.Store.YES,   
            Field.Index.TOKENIZED);
   // doc.add(Field.UnStored("body", handler.getDocument(new FileInputStream(f))));
    Field FieldName= new Field("filename", f.getCanonicalPath(), Field.Store.YES, Field.Index.UN_TOKENIZED); 
    doc.add(FieldBody);
    doc.add(FieldName);
    //doc.add(Field.Keyword("filename", f.getCanonicalPath()));
    writer.addDocument(doc);
  }
}