testqueryparser.java

Lucene a java open-source SearchEngine Framework

    assertEscapedQueryEquals("a:b+c", a, "a\\:b\\+c");
    assertEscapedQueryEquals("a:b:c", a, "a\\:b\\:c");
    assertEscapedQueryEquals("a:b\\c", a, "a\\:b\\\\c");

    assertEscapedQueryEquals("a:b-c*", a, "a\\:b\\-c\\*");
    assertEscapedQueryEquals("a:b+c*", a, "a\\:b\\+c\\*");
    assertEscapedQueryEquals("a:b:c*", a, "a\\:b\\:c\\*");

    assertEscapedQueryEquals("a:b\\\\c*", a, "a\\:b\\\\\\\\c\\*");

    assertEscapedQueryEquals("a:b-?c", a, "a\\:b\\-\\?c");
    assertEscapedQueryEquals("a:b+?c", a, "a\\:b\\+\\?c");
    assertEscapedQueryEquals("a:b:?c", a, "a\\:b\\:\\?c");

    assertEscapedQueryEquals("a:b?c", a, "a\\:b\\?c");

    assertEscapedQueryEquals("a:b-c~", a, "a\\:b\\-c\\~");
    assertEscapedQueryEquals("a:b+c~", a, "a\\:b\\+c\\~");
    assertEscapedQueryEquals("a:b:c~", a, "a\\:b\\:c\\~");
    assertEscapedQueryEquals("a:b\\c~", a, "a\\:b\\\\c\\~");

    assertEscapedQueryEquals("[ a - TO a+ ]", null, "\\[ a \\- TO a\\+ \\]");
    assertEscapedQueryEquals("[ a : TO a~ ]", null, "\\[ a \\: TO a\\~ \\]");
    assertEscapedQueryEquals("[ a\\ TO a* ]", null, "\\[ a\\\\ TO a\\* \\]");
    
    // LUCENE-881
    assertEscapedQueryEquals("|| abc ||", a, "\\|\\| abc \\|\\|");
    assertEscapedQueryEquals("&& abc &&", a, "\\&\\& abc \\&\\&");
  }
  
  public void testTabNewlineCarriageReturn()
    throws Exception {
    assertQueryEqualsDOA("+weltbank +worlbank", null,
      "+weltbank +worlbank");

    assertQueryEqualsDOA("+weltbank\n+worlbank", null,
      "+weltbank +worlbank");
    assertQueryEqualsDOA("weltbank \n+worlbank", null,
      "+weltbank +worlbank");
    assertQueryEqualsDOA("weltbank \n +worlbank", null,
      "+weltbank +worlbank");

    assertQueryEqualsDOA("+weltbank\r+worlbank", null,
      "+weltbank +worlbank");
    assertQueryEqualsDOA("weltbank \r+worlbank", null,
      "+weltbank +worlbank");
    assertQueryEqualsDOA("weltbank \r +worlbank", null,
      "+weltbank +worlbank");

    assertQueryEqualsDOA("+weltbank\r\n+worlbank", null,
      "+weltbank +worlbank");
    assertQueryEqualsDOA("weltbank \r\n+worlbank", null,
      "+weltbank +worlbank");
    assertQueryEqualsDOA("weltbank \r\n +worlbank", null,
      "+weltbank +worlbank");
    assertQueryEqualsDOA("weltbank \r \n +worlbank", null,
      "+weltbank +worlbank");

    assertQueryEqualsDOA("+weltbank\t+worlbank", null,
      "+weltbank +worlbank");
    assertQueryEqualsDOA("weltbank \t+worlbank", null,
      "+weltbank +worlbank");
    assertQueryEqualsDOA("weltbank \t +worlbank", null,
      "+weltbank +worlbank");
  }

  public void testSimpleDAO()
    throws Exception {
    assertQueryEqualsDOA("term term term", null, "+term +term +term");
    assertQueryEqualsDOA("term +term term", null, "+term +term +term");
    assertQueryEqualsDOA("term term +term", null, "+term +term +term");
    assertQueryEqualsDOA("term +term +term", null, "+term +term +term");
    assertQueryEqualsDOA("-term term term", null, "-term +term +term");
  }

  public void testBoost()
    throws Exception {
    StandardAnalyzer oneStopAnalyzer = new StandardAnalyzer(new String[]{"on"});
    QueryParser qp = new QueryParser("field", oneStopAnalyzer);
    Query q = qp.parse("on^1.0");
    assertNotNull(q);
    q = qp.parse("\"hello\"^2.0");
    assertNotNull(q);
    assertEquals(q.getBoost(), (float) 2.0, (float) 0.5);
    q = qp.parse("hello^2.0");
    assertNotNull(q);
    assertEquals(q.getBoost(), (float) 2.0, (float) 0.5);
    q = qp.parse("\"on\"^1.0");
    assertNotNull(q);

    QueryParser qp2 = new QueryParser("field", new StandardAnalyzer());
    q = qp2.parse("the^3");
    // "the" is a stop word so the result is an empty query:
    assertNotNull(q);
    assertEquals("", q.toString());
    assertEquals(1.0f, q.getBoost(), 0.01f);
  }

  public void assertParseException(String queryString) throws Exception {
    try {
      Query q = getQuery(queryString, null);
    } catch (ParseException expected) {
      return;
    }
    fail("ParseException expected, not thrown");
  }
       
  public void testException() throws Exception {
    assertParseException("\"some phrase");
    assertParseException("(foo bar");
    assertParseException("foo bar))");
    assertParseException("field:term:with:colon some more terms");
    assertParseException("(sub query)^5.0^2.0 plus more");
    assertParseException("secret AND illegal) AND access:confidential");
  }
  

  public void testCustomQueryParserWildcard() {
    try {
      new QPTestParser("contents", new WhitespaceAnalyzer()).parse("a?t");
      fail("Wildcard queries should not be allowed");
    } catch (ParseException expected) {
      // expected exception
    }
  }

  public void testCustomQueryParserFuzzy() throws Exception {
    try {
      new QPTestParser("contents", new WhitespaceAnalyzer()).parse("xunit~");
      fail("Fuzzy queries should not be allowed");
    } catch (ParseException expected) {
      // expected exception
    }
  }

  public void testBooleanQuery() throws Exception {
    BooleanQuery.setMaxClauseCount(2);
    try {
      QueryParser qp = new QueryParser("field", new WhitespaceAnalyzer());
      qp.parse("one two three");
      fail("ParseException expected due to too many boolean clauses");
    } catch (ParseException expected) {
      // too many boolean clauses, so ParseException is expected
    }
  }

  /**
   * This test differs from TestPrecedenceQueryParser
   */
  public void testPrecedence() throws Exception {
    QueryParser qp = new QueryParser("field", new WhitespaceAnalyzer());
    Query query1 = qp.parse("A AND B OR C AND D");
    Query query2 = qp.parse("+A +B +C +D");
    assertEquals(query1, query2);
  }

  public void testLocalDateFormat() throws IOException, ParseException {
    RAMDirectory ramDir = new RAMDirectory();
    IndexWriter iw = new IndexWriter(ramDir, new WhitespaceAnalyzer(), true);
    addDateDoc("a", 2005, 12, 2, 10, 15, 33, iw);
    addDateDoc("b", 2005, 12, 4, 22, 15, 00, iw);
    iw.close();
    IndexSearcher is = new IndexSearcher(ramDir);
    assertHits(1, "[12/1/2005 TO 12/3/2005]", is);
    assertHits(2, "[12/1/2005 TO 12/4/2005]", is);
    assertHits(1, "[12/3/2005 TO 12/4/2005]", is);
    assertHits(1, "{12/1/2005 TO 12/3/2005}", is);
    assertHits(1, "{12/1/2005 TO 12/4/2005}", is);
    assertHits(0, "{12/3/2005 TO 12/4/2005}", is);
    is.close();
  }

  public void testStarParsing() throws Exception {
    final int[] type = new int[1];
    QueryParser qp = new QueryParser("field", new WhitespaceAnalyzer()) {
      protected Query getWildcardQuery(String field, String termStr) throws ParseException {
        // override error checking of superclass
        type[0]=1;
        return new TermQuery(new Term(field,termStr));
      }
      protected Query getPrefixQuery(String field, String termStr) throws ParseException {
        // override error checking of superclass
        type[0]=2;        
        return new TermQuery(new Term(field,termStr));
      }

      protected Query getFieldQuery(String field, String queryText) throws ParseException {
        type[0]=3;
        return super.getFieldQuery(field, queryText);
      }
    };

    TermQuery tq;

    tq = (TermQuery)qp.parse("foo:zoo*");
    assertEquals("zoo",tq.getTerm().text());
    assertEquals(2,type[0]);

    tq = (TermQuery)qp.parse("foo:zoo*^2");
    assertEquals("zoo",tq.getTerm().text());
    assertEquals(2,type[0]);
    assertEquals(tq.getBoost(),2,0);

    tq = (TermQuery)qp.parse("foo:*");
    assertEquals("*",tq.getTerm().text());
    assertEquals(1,type[0]);  // could be a valid prefix query in the future too

    tq = (TermQuery)qp.parse("foo:*^2");
    assertEquals("*",tq.getTerm().text());
    assertEquals(1,type[0]);
    assertEquals(tq.getBoost(),2,0);    

    tq = (TermQuery)qp.parse("*:foo");
    assertEquals("*",tq.getTerm().field());
    assertEquals("foo",tq.getTerm().text());
    assertEquals(3,type[0]);

    tq = (TermQuery)qp.parse("*:*");
    assertEquals("*",tq.getTerm().field());
    assertEquals("*",tq.getTerm().text());
    assertEquals(1,type[0]);  // could be handled as a prefix query in the future

     tq = (TermQuery)qp.parse("(*:*)");
    assertEquals("*",tq.getTerm().field());
    assertEquals("*",tq.getTerm().text());
    assertEquals(1,type[0]);

  }

  public void testStopwords() throws Exception {
    QueryParser qp = new QueryParser("a", new StopAnalyzer(new String[]{"the", "foo"}));
    Query result = qp.parse("a:the OR a:foo");
    assertNotNull("result is null and it shouldn't be", result);
    assertTrue("result is not a BooleanQuery", result instanceof BooleanQuery);
    assertTrue(((BooleanQuery) result).clauses().size() + " does not equal: " + 0, ((BooleanQuery) result).clauses().size() == 0);
    result = qp.parse("a:woo OR a:the");
    assertNotNull("result is null and it shouldn't be", result);
    assertTrue("result is not a TermQuery", result instanceof TermQuery);
    result = qp.parse("(fieldX:xxxxx OR fieldy:xxxxxxxx)^2 AND (fieldx:the OR fieldy:foo)");
    assertNotNull("result is null and it shouldn't be", result);
    assertTrue("result is not a BooleanQuery", result instanceof BooleanQuery);
    System.out.println("Result: " + result);
    assertTrue(((BooleanQuery) result).clauses().size() + " does not equal: " + 2, ((BooleanQuery) result).clauses().size() == 2);
  }

  public void testPositionIncrement() throws Exception {
    boolean dflt = StopFilter.getEnablePositionIncrementsDefault();
    StopFilter.setEnablePositionIncrementsDefault(true);
    try {
      QueryParser qp = new QueryParser("a", new StopAnalyzer(new String[]{"the", "in", "are", "this"}));
      qp.setEnablePositionIncrements(true);
      String qtxt = "\"the words in poisitions pos02578 are stopped in this phrasequery\"";
      //               0         2                      5           7  8
      int expectedPositions[] = {1,3,4,6,9};
      PhraseQuery pq = (PhraseQuery) qp.parse(qtxt);
      //System.out.println("Query text: "+qtxt);
      //System.out.println("Result: "+pq);
      Term t[] = pq.getTerms();
      int pos[] = pq.getPositions();
      for (int i = 0; i < t.length; i++) {
        //System.out.println(i+". "+t[i]+"  pos: "+pos[i]);
        assertEquals("term "+i+" = "+t[i]+" has wrong term-position!",expectedPositions[i],pos[i]);
      }
    } finally {
      StopFilter.setEnablePositionIncrementsDefault(dflt);
    }
  }

  public void testMatchAllDocs() throws Exception {
    QueryParser qp = new QueryParser("field", new WhitespaceAnalyzer());
    assertEquals(new MatchAllDocsQuery(), qp.parse("*:*"));
    assertEquals(new MatchAllDocsQuery(), qp.parse("(*:*)"));
    BooleanQuery bq = (BooleanQuery)qp.parse("+*:* -*:*");
    assertTrue(bq.getClauses()[0].getQuery() instanceof MatchAllDocsQuery);
    assertTrue(bq.getClauses()[1].getQuery() instanceof MatchAllDocsQuery);
  }
  
  private void assertHits(int expected, String query, IndexSearcher is) throws ParseException, IOException {
    QueryParser qp = new QueryParser("date", new WhitespaceAnalyzer());
    qp.setLocale(Locale.ENGLISH);
    Query q = qp.parse(query);
    Hits hits = is.search(q);
    assertEquals(expected, hits.length());
  }

  private static void addDateDoc(String content, int year, int month,
      int day, int hour, int minute, int second, IndexWriter iw) throws IOException {
    Document d = new Document();
    d.add(new Field("f", content, Field.Store.YES, Field.Index.TOKENIZED));
    Calendar cal = Calendar.getInstance();
    cal.set(year, month-1, day, hour, minute, second);
    d.add(new Field("date", DateField.dateToString(cal.getTime()), Field.Store.YES, Field.Index.UN_TOKENIZED));
    iw.addDocument(d);
  }

  public void tearDown() throws Exception {
    super.tearDown();
    BooleanQuery.setMaxClauseCount(originalMaxClauses);
  }

}