testsort.java

Lucene a java open-source SearchEngine Framework

	// test using various international locales with accented characters
	// (which sort differently depending on locale)
	public void testInternationalSort() throws Exception {
		sort.setSort (new SortField ("i18n", Locale.US));
		assertMatches (full, queryY, sort, "BFJDH");

		sort.setSort (new SortField ("i18n", new Locale("sv", "se")));
		assertMatches (full, queryY, sort, "BJDFH");

		sort.setSort (new SortField ("i18n", new Locale("da", "dk")));
		assertMatches (full, queryY, sort, "BJDHF");

		sort.setSort (new SortField ("i18n", Locale.US));
		assertMatches (full, queryX, sort, "ECAGI");

		sort.setSort (new SortField ("i18n", Locale.FRANCE));
		assertMatches (full, queryX, sort, "EACGI");
	}
    
    // Test the MultiSearcher's ability to preserve locale-sensitive ordering
    // by wrapping it around a single searcher
	public void testInternationalMultiSearcherSort() throws Exception {
		Searcher multiSearcher = new MultiSearcher (new Searchable[] { full });
		
		sort.setSort (new SortField ("i18n", new Locale("sv", "se")));
		assertMatches (multiSearcher, queryY, sort, "BJDFH");
		
		sort.setSort (new SortField ("i18n", Locale.US));
		assertMatches (multiSearcher, queryY, sort, "BFJDH");
		
		sort.setSort (new SortField ("i18n", new Locale("da", "dk")));
		assertMatches (multiSearcher, queryY, sort, "BJDHF");
	} 
    
	// test a custom sort function
	public void testCustomSorts() throws Exception {
		sort.setSort (new SortField ("custom", SampleComparable.getComparatorSource()));
		assertMatches (full, queryX, sort, "CAIEG");
		sort.setSort (new SortField ("custom", SampleComparable.getComparatorSource(), true));
		assertMatches (full, queryY, sort, "HJDBF");
		SortComparator custom = SampleComparable.getComparator();
		sort.setSort (new SortField ("custom", custom));
		assertMatches (full, queryX, sort, "CAIEG");
		sort.setSort (new SortField ("custom", custom, true));
		assertMatches (full, queryY, sort, "HJDBF");
	}

	// test a variety of sorts using more than one searcher
	public void testMultiSort() throws Exception {
		MultiSearcher searcher = new MultiSearcher (new Searchable[] { searchX, searchY });
		runMultiSorts (searcher);
	}

	// test a variety of sorts using a parallel multisearcher
	public void testParallelMultiSort() throws Exception {
		Searcher searcher = new ParallelMultiSearcher (new Searchable[] { searchX, searchY });
		runMultiSorts (searcher);
	}

	// test a variety of sorts using a remote searcher
	public void testRemoteSort() throws Exception {
		Searchable searcher = getRemote();
		MultiSearcher multi = new MultiSearcher (new Searchable[] { searcher });
		runMultiSorts (multi);
	}

	// test custom search when remote
	public void testRemoteCustomSort() throws Exception {
		Searchable searcher = getRemote();
		MultiSearcher multi = new MultiSearcher (new Searchable[] { searcher });
		sort.setSort (new SortField ("custom", SampleComparable.getComparatorSource()));
		assertMatches (multi, queryX, sort, "CAIEG");
		sort.setSort (new SortField ("custom", SampleComparable.getComparatorSource(), true));
		assertMatches (multi, queryY, sort, "HJDBF");
		SortComparator custom = SampleComparable.getComparator();
		sort.setSort (new SortField ("custom", custom));
		assertMatches (multi, queryX, sort, "CAIEG");
		sort.setSort (new SortField ("custom", custom, true));
		assertMatches (multi, queryY, sort, "HJDBF");
	}

	// test that the relevancy scores are the same even if
	// hits are sorted
	public void testNormalizedScores() throws Exception {

		// capture relevancy scores
		HashMap scoresX = getScores (full.search (queryX));
		HashMap scoresY = getScores (full.search (queryY));
		HashMap scoresA = getScores (full.search (queryA));

		// we'll test searching locally, remote and multi
		MultiSearcher remote = new MultiSearcher (new Searchable[] { getRemote() });
		MultiSearcher multi  = new MultiSearcher (new Searchable[] { searchX, searchY });

		// change sorting and make sure relevancy stays the same

		sort = new Sort();
		assertSameValues (scoresX, getScores(full.search(queryX,sort)));
		assertSameValues (scoresX, getScores(remote.search(queryX,sort)));
		assertSameValues (scoresX, getScores(multi.search(queryX,sort)));
		assertSameValues (scoresY, getScores(full.search(queryY,sort)));
		assertSameValues (scoresY, getScores(remote.search(queryY,sort)));
		assertSameValues (scoresY, getScores(multi.search(queryY,sort)));
		assertSameValues (scoresA, getScores(full.search(queryA,sort)));
		assertSameValues (scoresA, getScores(remote.search(queryA,sort)));
		assertSameValues (scoresA, getScores(multi.search(queryA,sort)));

		sort.setSort(SortField.FIELD_DOC);
		assertSameValues (scoresX, getScores(full.search(queryX,sort)));
		assertSameValues (scoresX, getScores(remote.search(queryX,sort)));
		assertSameValues (scoresX, getScores(multi.search(queryX,sort)));
		assertSameValues (scoresY, getScores(full.search(queryY,sort)));
		assertSameValues (scoresY, getScores(remote.search(queryY,sort)));
		assertSameValues (scoresY, getScores(multi.search(queryY,sort)));
		assertSameValues (scoresA, getScores(full.search(queryA,sort)));
		assertSameValues (scoresA, getScores(remote.search(queryA,sort)));
		assertSameValues (scoresA, getScores(multi.search(queryA,sort)));

		sort.setSort ("int");
		assertSameValues (scoresX, getScores(full.search(queryX,sort)));
		assertSameValues (scoresX, getScores(remote.search(queryX,sort)));
		assertSameValues (scoresX, getScores(multi.search(queryX,sort)));
		assertSameValues (scoresY, getScores(full.search(queryY,sort)));
		assertSameValues (scoresY, getScores(remote.search(queryY,sort)));
		assertSameValues (scoresY, getScores(multi.search(queryY,sort)));
		assertSameValues (scoresA, getScores(full.search(queryA,sort)));
		assertSameValues (scoresA, getScores(remote.search(queryA,sort)));
		assertSameValues (scoresA, getScores(multi.search(queryA,sort)));

		sort.setSort ("float");
		assertSameValues (scoresX, getScores(full.search(queryX,sort)));
		assertSameValues (scoresX, getScores(remote.search(queryX,sort)));
		assertSameValues (scoresX, getScores(multi.search(queryX,sort)));
		assertSameValues (scoresY, getScores(full.search(queryY,sort)));
		assertSameValues (scoresY, getScores(remote.search(queryY,sort)));
		assertSameValues (scoresY, getScores(multi.search(queryY,sort)));
		assertSameValues (scoresA, getScores(full.search(queryA,sort)));
		assertSameValues (scoresA, getScores(remote.search(queryA,sort)));
		assertSameValues (scoresA, getScores(multi.search(queryA,sort)));

		sort.setSort ("string");
		assertSameValues (scoresX, getScores(full.search(queryX,sort)));
		assertSameValues (scoresX, getScores(remote.search(queryX,sort)));
		assertSameValues (scoresX, getScores(multi.search(queryX,sort)));
		assertSameValues (scoresY, getScores(full.search(queryY,sort)));
		assertSameValues (scoresY, getScores(remote.search(queryY,sort)));
		assertSameValues (scoresY, getScores(multi.search(queryY,sort)));
		assertSameValues (scoresA, getScores(full.search(queryA,sort)));
		assertSameValues (scoresA, getScores(remote.search(queryA,sort)));
		assertSameValues (scoresA, getScores(multi.search(queryA,sort)));

		sort.setSort (new String[] {"int","float"});
		assertSameValues (scoresX, getScores(full.search(queryX,sort)));
		assertSameValues (scoresX, getScores(remote.search(queryX,sort)));
		assertSameValues (scoresX, getScores(multi.search(queryX,sort)));
		assertSameValues (scoresY, getScores(full.search(queryY,sort)));
		assertSameValues (scoresY, getScores(remote.search(queryY,sort)));
		assertSameValues (scoresY, getScores(multi.search(queryY,sort)));
		assertSameValues (scoresA, getScores(full.search(queryA,sort)));
		assertSameValues (scoresA, getScores(remote.search(queryA,sort)));
		assertSameValues (scoresA, getScores(multi.search(queryA,sort)));

		sort.setSort (new SortField[] { new SortField ("int", true), new SortField (null, SortField.DOC, true) });
		assertSameValues (scoresX, getScores(full.search(queryX,sort)));
		assertSameValues (scoresX, getScores(remote.search(queryX,sort)));
		assertSameValues (scoresX, getScores(multi.search(queryX,sort)));
		assertSameValues (scoresY, getScores(full.search(queryY,sort)));
		assertSameValues (scoresY, getScores(remote.search(queryY,sort)));
		assertSameValues (scoresY, getScores(multi.search(queryY,sort)));
		assertSameValues (scoresA, getScores(full.search(queryA,sort)));
		assertSameValues (scoresA, getScores(remote.search(queryA,sort)));
		assertSameValues (scoresA, getScores(multi.search(queryA,sort)));

		sort.setSort (new String[] {"float","string"});
		assertSameValues (scoresX, getScores(full.search(queryX,sort)));
		assertSameValues (scoresX, getScores(remote.search(queryX,sort)));
		assertSameValues (scoresX, getScores(multi.search(queryX,sort)));
		assertSameValues (scoresY, getScores(full.search(queryY,sort)));
		assertSameValues (scoresY, getScores(remote.search(queryY,sort)));
		assertSameValues (scoresY, getScores(multi.search(queryY,sort)));
		assertSameValues (scoresA, getScores(full.search(queryA,sort)));
		assertSameValues (scoresA, getScores(remote.search(queryA,sort)));
		assertSameValues (scoresA, getScores(multi.search(queryA,sort)));

	}

  public void testTopDocsScores() throws Exception {

    // There was previously a bug in FieldSortedHitQueue.maxscore when only a single
    // doc was added.  That is what the following tests for.
    Sort sort = new Sort();
    int nDocs=10;

    // try to pick a query that will result in an unnormalized
    // score greater than 1 to test for correct normalization
    final TopDocs docs1 = full.search(queryE,null,nDocs,sort);

    // a filter that only allows through the first hit
    Filter filt = new Filter() {
      public BitSet bits(IndexReader reader) throws IOException {
        BitSet bs = new BitSet(reader.maxDoc());
        bs.set(docs1.scoreDocs[0].doc);
        return bs;
      }
    };

    TopDocs docs2 = full.search(queryE, filt, nDocs, sort);
    
    assertEquals(docs1.scoreDocs[0].score, docs2.scoreDocs[0].score, 1e-6);
  }


  // runs a variety of sorts useful for multisearchers
	private void runMultiSorts (Searcher multi) throws Exception {
		sort.setSort (SortField.FIELD_DOC);
		assertMatchesPattern (multi, queryA, sort, "[AB]{2}[CD]{2}[EF]{2}[GH]{2}[IJ]{2}");

		sort.setSort (new SortField ("int", SortField.INT));
		assertMatchesPattern (multi, queryA, sort, "IDHFGJ[ABE]{3}C");

		sort.setSort (new SortField[] {new SortField ("int", SortField.INT), SortField.FIELD_DOC});
		assertMatchesPattern (multi, queryA, sort, "IDHFGJ[AB]{2}EC");

		sort.setSort ("int");
		assertMatchesPattern (multi, queryA, sort, "IDHFGJ[AB]{2}EC");

		sort.setSort (new SortField[] {new SortField ("float", SortField.FLOAT), SortField.FIELD_DOC});
		assertMatchesPattern (multi, queryA, sort, "GDHJ[CI]{2}EFAB");

		sort.setSort ("float");
		assertMatchesPattern (multi, queryA, sort, "GDHJ[CI]{2}EFAB");

		sort.setSort ("string");
		assertMatches (multi, queryA, sort, "DJAIHGFEBC");

		sort.setSort ("int", true);
		assertMatchesPattern (multi, queryA, sort, "C[AB]{2}EJGFHDI");

		sort.setSort ("float", true);
		assertMatchesPattern (multi, queryA, sort, "BAFE[IC]{2}JHDG");

		sort.setSort ("string", true);
		assertMatches (multi, queryA, sort, "CBEFGHIAJD");

		sort.setSort (new SortField[] { new SortField ("string", Locale.US) });
		assertMatches (multi, queryA, sort, "DJAIHGFEBC");

		sort.setSort (new SortField[] { new SortField ("string", Locale.US, true) });
		assertMatches (multi, queryA, sort, "CBEFGHIAJD");

		sort.setSort (new String[] {"int","float"});
		assertMatches (multi, queryA, sort, "IDHFGJEABC");

		sort.setSort (new String[] {"float","string"});
		assertMatches (multi, queryA, sort, "GDHJICEFAB");

		sort.setSort ("int");
		assertMatches (multi, queryF, sort, "IZJ");

		sort.setSort ("int", true);
		assertMatches (multi, queryF, sort, "JZI");

		sort.setSort ("float");
		assertMatches (multi, queryF, sort, "ZJI");

		sort.setSort ("string");
		assertMatches (multi, queryF, sort, "ZJI");

		sort.setSort ("string", true);
		assertMatches (multi, queryF, sort, "IJZ");
	}

	// make sure the documents returned by the search match the expected list
	private void assertMatches (Searcher searcher, Query query, Sort sort, String expectedResult)
	throws IOException {
		Hits result = searcher.search (query, sort);
		StringBuffer buff = new StringBuffer(10);
		int n = result.length();
		for (int i=0; i<n; ++i) {
			Document doc = result.doc(i);
			String[] v = doc.getValues("tracer");
			for (int j=0; j<v.length; ++j) {
				buff.append (v[j]);
			}
		}
		assertEquals (expectedResult, buff.toString());
	}

	// make sure the documents returned by the search match the expected list pattern
	private void assertMatchesPattern (Searcher searcher, Query query, Sort sort, String pattern)
	throws IOException {
		Hits result = searcher.search (query, sort);
		StringBuffer buff = new StringBuffer(10);
		int n = result.length();
		for (int i=0; i<n; ++i) {
			Document doc = result.doc(i);
			String[] v = doc.getValues("tracer");
			for (int j=0; j<v.length; ++j) {
				buff.append (v[j]);
			}
		}
		// System.out.println ("matching \""+buff+"\" against pattern \""+pattern+"\"");
		assertTrue (Pattern.compile(pattern).matcher(buff.toString()).matches());
	}

	private HashMap getScores (Hits hits)
	throws IOException {
		HashMap scoreMap = new HashMap();
		int n = hits.length();
		for (int i=0; i<n; ++i) {
			Document doc = hits.doc(i);
			String[] v = doc.getValues("tracer");
			assertEquals (v.length, 1);
			scoreMap.put (v[0], new Float(hits.score(i)));
		}
		return scoreMap;
	}

	// make sure all the values in the maps match
	private void assertSameValues (HashMap m1, HashMap m2) {
		int n = m1.size();
		int m = m2.size();
		assertEquals (n, m);
		Iterator iter = m1.keySet().iterator();
		while (iter.hasNext()) {
			Object key = iter.next();
      Object o1 = m1.get(key);
      Object o2 = m2.get(key);
      if (o1 instanceof Float) {
        assertEquals(((Float)o1).floatValue(), ((Float)o2).floatValue(), 1e-6);
      } else {
        assertEquals (m1.get(key), m2.get(key));
      }
		}
	}

	private Searchable getRemote () throws Exception {
		try {
			return lookupRemote ();
		} catch (Throwable e) {
			startServer ();
			return lookupRemote ();
		}
	}

	private Searchable lookupRemote () throws Exception {
		return (Searchable) Naming.lookup ("//localhost/SortedSearchable");
	}

	private void startServer () throws Exception {
		// construct an index
		Searcher local = getFullIndex();
		// local.search (queryA, new Sort());

		// publish it
		Registry reg = LocateRegistry.createRegistry (1099);
		RemoteSearchable impl = new RemoteSearchable (local);
		Naming.rebind ("//localhost/SortedSearchable", impl);
	}

}