highlightertest.java

lucene2.2.0版本

package org.apache.lucene.search.highlight;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.*;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;

import junit.framework.TestCase;

import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.FilteredQuery;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MultiSearcher;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.RangeFilter;
import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.store.RAMDirectory;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;

/**
 * JUnit Test for Highlighter class.
 * @author [e-mail]
 */
public class HighlighterTest extends TestCase implements Formatter
{
	private IndexReader reader;
	private static final String FIELD_NAME = "contents";
	private Query query;
	RAMDirectory ramDir;
	public Searcher searcher = null;
	public Hits hits = null;
	int numHighlights = 0;
	Analyzer analyzer=new StandardAnalyzer();

	String texts[] =
		{
			"Hello this is a piece of text that is very long and contains too much preamble and the meat is really here which says kennedy has been shot",
			"This piece of text refers to Kennedy at the beginning then has a longer piece of text that is very long in the middle and finally ends with another reference to Kennedy",
			"JFK has been shot",
			"John Kennedy has been shot",
			"This text has a typo in referring to Keneddy" };

	/**
	 * Constructor for HighlightExtractorTest.
	 * @param arg0
	 */
	public HighlighterTest(String arg0)
	{
		super(arg0);
	}

	public void testSimpleHighlighter() throws Exception
	{
		doSearching("Kennedy");
		Highlighter highlighter =	new Highlighter(new QueryScorer(query));
		highlighter.setTextFragmenter(new SimpleFragmenter(40));
		int maxNumFragmentsRequired = 2;
		for (int i = 0; i < hits.length(); i++)
		{
			String text = hits.doc(i).get(FIELD_NAME);
			TokenStream tokenStream=analyzer.tokenStream(FIELD_NAME,new StringReader(text));

			String result =
				highlighter.getBestFragments(tokenStream,text,maxNumFragmentsRequired, "...");
			System.out.println("\t" + result);
		}
		//Not sure we can assert anything here - just running to check we dont throw any exceptions
	}



	public void testGetBestFragmentsSimpleQuery() throws Exception
	{
		doSearching("Kennedy");
		doStandardHighlights();
		assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 4);
	}
	public void testGetFuzzyFragments() throws Exception
	{
		doSearching("Kinnedy~");
		doStandardHighlights();
		assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 5);
	}

	public void testGetWildCardFragments() throws Exception
	{
		doSearching("K?nnedy");
		doStandardHighlights();
		assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 4);
	}
	public void testGetMidWildCardFragments() throws Exception
	{
		doSearching("K*dy");
		doStandardHighlights();
		assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 5);
	}
	public void testGetRangeFragments() throws Exception
	{
		String queryString=FIELD_NAME + ":[kannedy TO kznnedy]"; 
		
		//Need to explicitly set the QueryParser property to use RangeQuery rather than RangeFilters
		QueryParser parser=new QueryParser(FIELD_NAME, new StandardAnalyzer());
		parser.setUseOldRangeQuery(true);
		query = parser.parse(queryString);
		doSearching(query);
		
		doStandardHighlights();
		assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 5);
	}

	public void testGetBestFragmentsPhrase() throws Exception
	{
		doSearching("\"John Kennedy\"");
		doStandardHighlights();
		//Currently highlights "John" and "Kennedy" separately
		assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 2);
	}
	public void testGetBestFragmentsSpan() throws Exception
	{
		SpanQuery clauses[]={
			new SpanTermQuery(new Term("contents","john")),
			new SpanTermQuery(new Term("contents","kennedy")),
			}; 
		
		SpanNearQuery snq=new SpanNearQuery(clauses,1,true);
		doSearching(snq);
		doStandardHighlights();
		//Currently highlights "John" and "Kennedy" separately
		assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 2);
	}

	public void testOffByOne() throws IOException 
	{
	    TermQuery query= new TermQuery( new Term( "data", "help" ));
	    Highlighter hg = new Highlighter(new SimpleHTMLFormatter(), new QueryScorer( query ));
	    hg.setTextFragmenter( new NullFragmenter() );

	    String match = null;
	    match = hg.getBestFragment( new StandardAnalyzer(), "data", "help me [54-65]");
	    assertEquals("<B>help</B> me [54-65]", match);
	} 	
	public void testGetBestFragmentsFilteredQuery() throws Exception
	{
		RangeFilter rf=new RangeFilter("contents","john","john",true,true);
		SpanQuery clauses[]={
				new SpanTermQuery(new Term("contents","john")),
				new SpanTermQuery(new Term("contents","kennedy")),
				}; 
		SpanNearQuery snq=new SpanNearQuery(clauses,1,true);
		FilteredQuery fq=new FilteredQuery(snq,rf);
		
		doSearching(fq);
		doStandardHighlights();
		//Currently highlights "John" and "Kennedy" separately
		assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 2);
	}
	public void testGetBestFragmentsFilteredPhraseQuery() throws Exception
	{
		RangeFilter rf=new RangeFilter("contents","john","john",true,true);
		PhraseQuery pq=new PhraseQuery();
		pq.add(new Term("contents","john"));
		pq.add(new  Term("contents","kennedy"));
		FilteredQuery fq=new FilteredQuery(pq,rf);
		
		doSearching(fq);
		doStandardHighlights();
		//Currently highlights "John" and "Kennedy" separately
		assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 2);
	}

	public void testGetBestFragmentsMultiTerm() throws Exception
	{
		doSearching("John Kenn*");
		doStandardHighlights();
		assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 5);
	}
	public void testGetBestFragmentsWithOr() throws Exception
	{
		doSearching("JFK OR Kennedy");
		doStandardHighlights();
		assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 5);
	}

	public void testGetBestSingleFragment() throws Exception
	{
		doSearching("Kennedy");
		Highlighter highlighter =new Highlighter(this,new QueryScorer(query));
		highlighter.setTextFragmenter(new SimpleFragmenter(40));

		for (int i = 0; i < hits.length(); i++)
		{
			String text = hits.doc(i).get(FIELD_NAME);
			TokenStream tokenStream=analyzer.tokenStream(FIELD_NAME,new StringReader(text));
			String result = highlighter.getBestFragment(tokenStream,text);
			System.out.println("\t" + result);
		}
		assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 4);

		numHighlights = 0;
		for (int i = 0; i < hits.length(); i++)
		{
    		String text = hits.doc(i).get(FIELD_NAME);
    		highlighter.getBestFragment(analyzer, FIELD_NAME,text);
		}
		assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 4);

		numHighlights = 0;
		for (int i = 0; i < hits.length(); i++)
		{
    		String text = hits.doc(i).get(FIELD_NAME);
    		highlighter.getBestFragments(analyzer,FIELD_NAME, text, 10);
		}
		assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 4);

	}

	public void testGetBestSingleFragmentWithWeights() throws Exception
	{
		WeightedTerm[]wTerms=new WeightedTerm[2];
		wTerms[0]=new WeightedTerm(10f,"hello");
		wTerms[1]=new WeightedTerm(1f,"kennedy");
		Highlighter highlighter =new Highlighter(new QueryScorer(wTerms));
		TokenStream tokenStream=analyzer.tokenStream(FIELD_NAME,new StringReader(texts[0]));
		highlighter.setTextFragmenter(new SimpleFragmenter(2));

		String result = highlighter.getBestFragment(tokenStream,texts[0]).trim();
		assertTrue("Failed to find best section using weighted terms. Found: ["+result+"]"
			, "<B>Hello</B>".equals(result));

		//readjust weights
		wTerms[1].setWeight(50f);
		tokenStream=analyzer.tokenStream(FIELD_NAME,new StringReader(texts[0]));
		highlighter =new Highlighter(new QueryScorer(wTerms));
		highlighter.setTextFragmenter(new SimpleFragmenter(2));

		result = highlighter.getBestFragment(tokenStream,texts[0]).trim();
		assertTrue("Failed to find best section using weighted terms. Found: "+result
			, "<B>kennedy</B>".equals(result));
	}
	
	
	// tests a "complex" analyzer that produces multiple 
	// overlapping tokens 
	public void testOverlapAnalyzer() throws Exception
	{
		HashMap synonyms = new HashMap();
		synonyms.put("football", "soccer,footie");
		Analyzer analyzer = new SynonymAnalyzer(synonyms);
		String srchkey = "football";

		String s = "football-soccer in the euro 2004 footie competition";
		QueryParser parser=new QueryParser("bookid",analyzer);
		Query query = parser.parse(srchkey);

		Highlighter highlighter = new Highlighter(new QueryScorer(query));
		TokenStream tokenStream =
			analyzer.tokenStream(null, new StringReader(s));
		// Get 3 best fragments and seperate with a "..."
		String result = highlighter.getBestFragments(tokenStream, s, 3, "...");
		String expectedResult="<B>football</B>-<B>soccer</B> in the euro 2004 <B>footie</B> competition";
		assertTrue("overlapping analyzer should handle highlights OK",expectedResult.equals(result));
	}


	public void testGetSimpleHighlight() throws Exception
	{
		doSearching("Kennedy");
		Highlighter highlighter =
			new Highlighter(this,new QueryScorer(query));

		for (int i = 0; i < hits.length(); i++)