testsort.java

Lucene a java open-source SearchEngine Framework

package org.apache.lucene.search;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;
import junit.textui.TestRunner;
import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.RAMDirectory;

import java.io.IOException;
import java.io.Serializable;
import java.rmi.Naming;
import java.rmi.registry.LocateRegistry;
import java.rmi.registry.Registry;
import java.util.BitSet;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Locale;
import java.util.regex.Pattern;

/**
 * Unit tests for sorting code.
 *
 * <p>Created: Feb 17, 2004 4:55:10 PM
 *
 * @author  Tim Jones (Nacimiento Software)
 * @since   lucene 1.4
 * @version $Id: TestSort.java 590530 2007-10-31 01:28:25Z gsingers $
 */

public class TestSort
extends TestCase
implements Serializable {

	private Searcher full;
	private Searcher searchX;
	private Searcher searchY;
	private Query queryX;
	private Query queryY;
	private Query queryA;
	private Query queryE;
	private Query queryF;
	private Query queryG;
	private Sort sort;


	public TestSort (String name) {
		super (name);
	}

	public static void main (String[] argv) {
		if (argv == null || argv.length < 1)
			TestRunner.run (suite());
		else if ("server".equals (argv[0])) {
			TestSort test = new TestSort (null);
			try {
				test.startServer();
				Thread.sleep (500000);
			} catch (Exception e) {
				System.out.println (e);
				e.printStackTrace();
			}
		}
	}

	public static Test suite() {
		return new TestSuite (TestSort.class);
	}


	// document data:
	// the tracer field is used to determine which document was hit
	// the contents field is used to search and sort by relevance
	// the int field to sort by int
	// the float field to sort by float
	// the string field to sort by string
    // the i18n field includes accented characters for testing locale-specific sorting
	private String[][] data = new String[][] {
	// tracer  contents         int            float           string   custom   i18n               long            double
	{   "A",   "x a",           "5",           "4f",           "c",     "A-3",   "p\u00EAche",      "10",           "-4.0"},//A
	{   "B",   "y a",           "5",           "3.4028235E38", "i",     "B-10",  "HAT",             "1000000000", "40.0"},//B
	{   "C",   "x a b c",       "2147483647",  "1.0",          "j",     "A-2",   "p\u00E9ch\u00E9", "99999999",   "40.00002343"},//C
	{   "D",   "y a b c",       "-1",          "0.0f",         "a",     "C-0",   "HUT",             String.valueOf(Long.MAX_VALUE),           String.valueOf(Double.MIN_VALUE)},//D
	{   "E",   "x a b c d",     "5",           "2f",           "h",     "B-8",   "peach",           String.valueOf(Long.MIN_VALUE),           String.valueOf(Double.MAX_VALUE)},//E
	{   "F",   "y a b c d",     "2",           "3.14159f",     "g",     "B-1",   "H\u00C5T",        "-44",           "343.034435444"},//F
	{   "G",   "x a b c d",     "3",           "-1.0",         "f",     "C-100", "sin",             "323254543543", "4.043544"},//G
  {   "H",   "y a b c d",     "0",           "1.4E-45",      "e",     "C-88",  "H\u00D8T",        "1023423423005","4.043545"},//H
	{   "I",   "x a b c d e f", "-2147483648", "1.0e+0",       "d",     "A-10",  "s\u00EDn",        "332422459999", "4.043546"},//I
	{   "J",   "y a b c d e f", "4",           ".5",           "b",     "C-7",   "HOT",             "34334543543",  "4.0000220343"},//J
	{   "W",   "g",             "1",           null,           null,    null,    null,              null,           null},
	{   "X",   "g",             "1",           "0.1",          null,    null,    null,              null,           null},
	{   "Y",   "g",             "1",           "0.2",          null,    null,    null,              null,           null},
	{   "Z",   "f g",           null,          null,           null,    null,    null,              null,           null}
	};

	// create an index of all the documents, or just the x, or just the y documents
	private Searcher getIndex (boolean even, boolean odd)
	throws IOException {
		RAMDirectory indexStore = new RAMDirectory ();
		IndexWriter writer = new IndexWriter (indexStore, new SimpleAnalyzer(), true);
		for (int i=0; i<data.length; ++i) {
			if (((i%2)==0 && even) || ((i%2)==1 && odd)) {
				Document doc = new Document();
				doc.add (new Field ("tracer",   data[i][0], Field.Store.YES, Field.Index.NO));
				doc.add (new Field ("contents", data[i][1], Field.Store.NO, Field.Index.TOKENIZED));
				if (data[i][2] != null) doc.add (new Field ("int",      data[i][2], Field.Store.NO, Field.Index.UN_TOKENIZED));
				if (data[i][3] != null) doc.add (new Field ("float",    data[i][3], Field.Store.NO, Field.Index.UN_TOKENIZED));
				if (data[i][4] != null) doc.add (new Field ("string",   data[i][4], Field.Store.NO, Field.Index.UN_TOKENIZED));
				if (data[i][5] != null) doc.add (new Field ("custom",   data[i][5], Field.Store.NO, Field.Index.UN_TOKENIZED));
				if (data[i][6] != null) doc.add (new Field ("i18n",     data[i][6], Field.Store.NO, Field.Index.UN_TOKENIZED));
        if (data[i][7] != null) doc.add (new Field ("long",     data[i][7], Field.Store.NO, Field.Index.UN_TOKENIZED));
        if (data[i][8] != null) doc.add (new Field ("double",     data[i][8], Field.Store.NO, Field.Index.UN_TOKENIZED));
        doc.setBoost(2);  // produce some scores above 1.0
				writer.addDocument (doc);
			}
		}
		writer.optimize ();
		writer.close ();
		return new IndexSearcher (indexStore);
	}

	private Searcher getFullIndex()
	throws IOException {
		return getIndex (true, true);
	}

	private Searcher getXIndex()
	throws IOException {
		return getIndex (true, false);
	}

	private Searcher getYIndex()
	throws IOException {
		return getIndex (false, true);
	}

	private Searcher getEmptyIndex()
	throws IOException {
		return getIndex (false, false);
	}

	public void setUp() throws Exception {
		full = getFullIndex();
		searchX = getXIndex();
		searchY = getYIndex();
		queryX = new TermQuery (new Term ("contents", "x"));
		queryY = new TermQuery (new Term ("contents", "y"));
		queryA = new TermQuery (new Term ("contents", "a"));
    queryE = new TermQuery (new Term ("contents", "e"));
		queryF = new TermQuery (new Term ("contents", "f"));
		queryG = new TermQuery (new Term ("contents", "g"));
		sort = new Sort();
	}

	// test the sorts by score and document number
	public void testBuiltInSorts() throws Exception {
		sort = new Sort();
		assertMatches (full, queryX, sort, "ACEGI");
		assertMatches (full, queryY, sort, "BDFHJ");

		sort.setSort(SortField.FIELD_DOC);
		assertMatches (full, queryX, sort, "ACEGI");
		assertMatches (full, queryY, sort, "BDFHJ");
	}

	// test sorts where the type of field is specified
	public void testTypedSort() throws Exception {
		sort.setSort (new SortField[] { new SortField ("int", SortField.INT), SortField.FIELD_DOC });
		assertMatches (full, queryX, sort, "IGAEC");
		assertMatches (full, queryY, sort, "DHFJB");

		sort.setSort (new SortField[] { new SortField ("float", SortField.FLOAT), SortField.FIELD_DOC });
		assertMatches (full, queryX, sort, "GCIEA");
		assertMatches (full, queryY, sort, "DHJFB");

    sort.setSort (new SortField[] { new SortField ("long", SortField.LONG), SortField.FIELD_DOC });
		assertMatches (full, queryX, sort, "EACGI");
		assertMatches (full, queryY, sort, "FBJHD");

    sort.setSort (new SortField[] { new SortField ("double", SortField.DOUBLE), SortField.FIELD_DOC });
		assertMatches (full, queryX, sort, "AGICE");
		assertMatches (full, queryY, sort, "DJHBF");

    sort.setSort (new SortField[] { new SortField ("string", SortField.STRING), SortField.FIELD_DOC });
		assertMatches (full, queryX, sort, "AIGEC");
		assertMatches (full, queryY, sort, "DJHFB");
	}

	// test sorts when there's nothing in the index
	public void testEmptyIndex() throws Exception {
		Searcher empty = getEmptyIndex();

		sort = new Sort();
		assertMatches (empty, queryX, sort, "");

		sort.setSort(SortField.FIELD_DOC);
		assertMatches (empty, queryX, sort, "");

		sort.setSort (new SortField[] { new SortField ("int", SortField.INT), SortField.FIELD_DOC });
		assertMatches (empty, queryX, sort, "");

		sort.setSort (new SortField[] { new SortField ("string", SortField.STRING, true), SortField.FIELD_DOC });
		assertMatches (empty, queryX, sort, "");

		sort.setSort (new SortField[] { new SortField ("float", SortField.FLOAT), new SortField ("string", SortField.STRING) });
		assertMatches (empty, queryX, sort, "");
	}

	// test sorts where the type of field is determined dynamically
	public void testAutoSort() throws Exception {
		sort.setSort("int");
		assertMatches (full, queryX, sort, "IGAEC");
		assertMatches (full, queryY, sort, "DHFJB");

		sort.setSort("float");
		assertMatches (full, queryX, sort, "GCIEA");
		assertMatches (full, queryY, sort, "DHJFB");

		sort.setSort("string");
		assertMatches (full, queryX, sort, "AIGEC");
		assertMatches (full, queryY, sort, "DJHFB");
	}

	// test sorts in reverse
	public void testReverseSort() throws Exception {
		sort.setSort (new SortField[] { new SortField (null, SortField.SCORE, true), SortField.FIELD_DOC });
		assertMatches (full, queryX, sort, "IEGCA");
		assertMatches (full, queryY, sort, "JFHDB");

		sort.setSort (new SortField (null, SortField.DOC, true));
		assertMatches (full, queryX, sort, "IGECA");
		assertMatches (full, queryY, sort, "JHFDB");

		sort.setSort ("int", true);
		assertMatches (full, queryX, sort, "CAEGI");
		assertMatches (full, queryY, sort, "BJFHD");

		sort.setSort ("float", true);
		assertMatches (full, queryX, sort, "AECIG");
		assertMatches (full, queryY, sort, "BFJHD");

		sort.setSort ("string", true);
		assertMatches (full, queryX, sort, "CEGIA");
		assertMatches (full, queryY, sort, "BFHJD");
	}

	// test sorting when the sort field is empty (undefined) for some of the documents
	public void testEmptyFieldSort() throws Exception {
		sort.setSort ("string");
		assertMatches (full, queryF, sort, "ZJI");

		sort.setSort ("string", true);
		assertMatches (full, queryF, sort, "IJZ");
		
		sort.setSort (new SortField ("i18n", Locale.ENGLISH));
		assertMatches (full, queryF, sort, "ZJI");
		
		sort.setSort (new SortField ("i18n", Locale.ENGLISH, true));
		assertMatches (full, queryF, sort, "IJZ");

		sort.setSort ("int");
		assertMatches (full, queryF, sort, "IZJ");

		sort.setSort ("int", true);
		assertMatches (full, queryF, sort, "JZI");

		sort.setSort ("float");
		assertMatches (full, queryF, sort, "ZJI");

		// using a nonexisting field as first sort key shouldn't make a difference:
		sort.setSort (new SortField[] { new SortField ("nosuchfield", SortField.STRING),
				new SortField ("float") });
		assertMatches (full, queryF, sort, "ZJI");

		sort.setSort ("float", true);
		assertMatches (full, queryF, sort, "IJZ");

		// When a field is null for both documents, the next SortField should be used.
                // Works for
		sort.setSort (new SortField[] { new SortField ("int"),
                                new SortField ("string", SortField.STRING),
				new SortField ("float") });
		assertMatches (full, queryG, sort, "ZWXY");

		// Reverse the last criterium to make sure the test didn't pass by chance
		sort.setSort (new SortField[] { new SortField ("int"),
                                new SortField ("string", SortField.STRING),
				new SortField ("float", true) });
		assertMatches (full, queryG, sort, "ZYXW");

		// Do the same for a MultiSearcher
		Searcher multiSearcher=new MultiSearcher (new Searchable[] { full });

		sort.setSort (new SortField[] { new SortField ("int"),
                                new SortField ("string", SortField.STRING),
				new SortField ("float") });
		assertMatches (multiSearcher, queryG, sort, "ZWXY");

		sort.setSort (new SortField[] { new SortField ("int"),
                                new SortField ("string", SortField.STRING),
				new SortField ("float", true) });
		assertMatches (multiSearcher, queryG, sort, "ZYXW");
		// Don't close the multiSearcher. it would close the full searcher too!

		// Do the same for a ParallelMultiSearcher
                Searcher parallelSearcher=new ParallelMultiSearcher (new Searchable[] { full });

		sort.setSort (new SortField[] { new SortField ("int"),
                                new SortField ("string", SortField.STRING),
				new SortField ("float") });
		assertMatches (parallelSearcher, queryG, sort, "ZWXY");

		sort.setSort (new SortField[] { new SortField ("int"),
                                new SortField ("string", SortField.STRING),
				new SortField ("float", true) });
		assertMatches (parallelSearcher, queryG, sort, "ZYXW");
		// Don't close the parallelSearcher. it would close the full searcher too!
	}

	// test sorts using a series of fields
	public void testSortCombos() throws Exception {
		sort.setSort (new String[] {"int","float"});
		assertMatches (full, queryX, sort, "IGEAC");

		sort.setSort (new SortField[] { new SortField ("int", true), new SortField (null, SortField.DOC, true) });
		assertMatches (full, queryX, sort, "CEAGI");

		sort.setSort (new String[] {"float","string"});
		assertMatches (full, queryX, sort, "GICEA");
	}

	// test using a Locale for sorting strings
	public void testLocaleSort() throws Exception {
		sort.setSort (new SortField[] { new SortField ("string", Locale.US) });
		assertMatches (full, queryX, sort, "AIGEC");
		assertMatches (full, queryY, sort, "DJHFB");

		sort.setSort (new SortField[] { new SortField ("string", Locale.US, true) });
		assertMatches (full, queryX, sort, "CEGIA");
		assertMatches (full, queryY, sort, "BFHJD");
	}