testindexwriterdelete.java

Lucene a java open-source SearchEngine Framework

package org.apache.lucene.index;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.IOException;
import java.util.Arrays;

import org.apache.lucene.util.LuceneTestCase;

import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockRAMDirectory;
import org.apache.lucene.store.RAMDirectory;

public class TestIndexWriterDelete extends LuceneTestCase {

  // test the simple case
  public void testSimpleCase() throws IOException {
    String[] keywords = { "1", "2" };
    String[] unindexed = { "Netherlands", "Italy" };
    String[] unstored = { "Amsterdam has lots of bridges",
        "Venice has lots of canals" };
    String[] text = { "Amsterdam", "Venice" };

    for(int pass=0;pass<2;pass++) {
      boolean autoCommit = (0==pass);

      Directory dir = new RAMDirectory();
      IndexWriter modifier = new IndexWriter(dir, autoCommit,
                                             new WhitespaceAnalyzer(), true);
      modifier.setUseCompoundFile(true);
      modifier.setMaxBufferedDeleteTerms(1);

      for (int i = 0; i < keywords.length; i++) {
        Document doc = new Document();
        doc.add(new Field("id", keywords[i], Field.Store.YES,
                          Field.Index.UN_TOKENIZED));
        doc.add(new Field("country", unindexed[i], Field.Store.YES,
                          Field.Index.NO));
        doc.add(new Field("contents", unstored[i], Field.Store.NO,
                          Field.Index.TOKENIZED));
        doc
          .add(new Field("city", text[i], Field.Store.YES,
                         Field.Index.TOKENIZED));
        modifier.addDocument(doc);
      }
      modifier.optimize();

      if (!autoCommit) {
        modifier.close();
      }

      Term term = new Term("city", "Amsterdam");
      int hitCount = getHitCount(dir, term);
      assertEquals(1, hitCount);
      if (!autoCommit) {
        modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
        modifier.setUseCompoundFile(true);
      }
      modifier.deleteDocuments(term);
      if (!autoCommit) {
        modifier.close();
      }
      hitCount = getHitCount(dir, term);
      assertEquals(0, hitCount);

      if (autoCommit) {
        modifier.close();
      }
      dir.close();
    }
  }

  // test when delete terms only apply to disk segments
  public void testNonRAMDelete() throws IOException {
    for(int pass=0;pass<2;pass++) {
      boolean autoCommit = (0==pass);

      Directory dir = new RAMDirectory();
      IndexWriter modifier = new IndexWriter(dir, autoCommit,
                                             new WhitespaceAnalyzer(), true);
      modifier.setMaxBufferedDocs(2);
      modifier.setMaxBufferedDeleteTerms(2);

      int id = 0;
      int value = 100;

      for (int i = 0; i < 7; i++) {
        addDoc(modifier, ++id, value);
      }
      modifier.flush();

      assertEquals(0, modifier.getNumBufferedDocuments());
      assertTrue(0 < modifier.getSegmentCount());

      if (!autoCommit) {
        modifier.close();
      }

      IndexReader reader = IndexReader.open(dir);
      assertEquals(7, reader.numDocs());
      reader.close();

      if (!autoCommit) {
        modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
        modifier.setMaxBufferedDocs(2);
        modifier.setMaxBufferedDeleteTerms(2);
      }

      modifier.deleteDocuments(new Term("value", String.valueOf(value)));
      modifier.deleteDocuments(new Term("value", String.valueOf(value)));

      if (!autoCommit) {
        modifier.close();
      }

      reader = IndexReader.open(dir);
      assertEquals(0, reader.numDocs());
      reader.close();
      if (autoCommit) {
        modifier.close();
      }
      dir.close();
    }
  }

  // test when delete terms only apply to ram segments
  public void testRAMDeletes() throws IOException {
    for(int pass=0;pass<2;pass++) {
      boolean autoCommit = (0==pass);
      Directory dir = new RAMDirectory();
      IndexWriter modifier = new IndexWriter(dir, autoCommit,
                                             new WhitespaceAnalyzer(), true);
      modifier.setMaxBufferedDocs(4);
      modifier.setMaxBufferedDeleteTerms(4);

      int id = 0;
      int value = 100;

      addDoc(modifier, ++id, value);
      modifier.deleteDocuments(new Term("value", String.valueOf(value)));
      addDoc(modifier, ++id, value);
      modifier.deleteDocuments(new Term("value", String.valueOf(value)));

      assertEquals(2, modifier.getNumBufferedDeleteTerms());
      assertEquals(1, modifier.getBufferedDeleteTermsSize());

      addDoc(modifier, ++id, value);
      assertEquals(0, modifier.getSegmentCount());
      modifier.flush();

      if (!autoCommit) {
        modifier.close();
      }

      IndexReader reader = IndexReader.open(dir);
      assertEquals(1, reader.numDocs());

      int hitCount = getHitCount(dir, new Term("id", String.valueOf(id)));
      assertEquals(1, hitCount);
      reader.close();
      if (autoCommit) {
        modifier.close();
      }
      dir.close();
    }
  }

  // test when delete terms apply to both disk and ram segments
  public void testBothDeletes() throws IOException {
    for(int pass=0;pass<2;pass++) {
      boolean autoCommit = (0==pass);

      Directory dir = new RAMDirectory();
      IndexWriter modifier = new IndexWriter(dir, autoCommit,
                                             new WhitespaceAnalyzer(), true);
      modifier.setMaxBufferedDocs(100);
      modifier.setMaxBufferedDeleteTerms(100);

      int id = 0;
      int value = 100;

      for (int i = 0; i < 5; i++) {
        addDoc(modifier, ++id, value);
      }

      value = 200;
      for (int i = 0; i < 5; i++) {
        addDoc(modifier, ++id, value);
      }
      modifier.flush();

      for (int i = 0; i < 5; i++) {
        addDoc(modifier, ++id, value);
      }
      modifier.deleteDocuments(new Term("value", String.valueOf(value)));

      modifier.flush();
      if (!autoCommit) {
        modifier.close();
      }

      IndexReader reader = IndexReader.open(dir);
      assertEquals(5, reader.numDocs());
      if (autoCommit) {
        modifier.close();
      }
    }
  }

  // test that batched delete terms are flushed together
  public void testBatchDeletes() throws IOException {
    for(int pass=0;pass<2;pass++) {
      boolean autoCommit = (0==pass);
      Directory dir = new RAMDirectory();
      IndexWriter modifier = new IndexWriter(dir, autoCommit,
                                             new WhitespaceAnalyzer(), true);
      modifier.setMaxBufferedDocs(2);
      modifier.setMaxBufferedDeleteTerms(2);

      int id = 0;
      int value = 100;

      for (int i = 0; i < 7; i++) {
        addDoc(modifier, ++id, value);
      }
      modifier.flush();
      if (!autoCommit) {
        modifier.close();
      }

      IndexReader reader = IndexReader.open(dir);
      assertEquals(7, reader.numDocs());
      reader.close();
      
      if (!autoCommit) {
        modifier = new IndexWriter(dir, autoCommit,
                                   new WhitespaceAnalyzer());
        modifier.setMaxBufferedDocs(2);
        modifier.setMaxBufferedDeleteTerms(2);
      }

      id = 0;
      modifier.deleteDocuments(new Term("id", String.valueOf(++id)));
      modifier.deleteDocuments(new Term("id", String.valueOf(++id)));

      if (!autoCommit) {
        modifier.close();
      }

      reader = IndexReader.open(dir);
      assertEquals(5, reader.numDocs());
      reader.close();

      Term[] terms = new Term[3];
      for (int i = 0; i < terms.length; i++) {
        terms[i] = new Term("id", String.valueOf(++id));
      }
      if (!autoCommit) {
        modifier = new IndexWriter(dir, autoCommit,
                                   new WhitespaceAnalyzer());
        modifier.setMaxBufferedDocs(2);
        modifier.setMaxBufferedDeleteTerms(2);
      }
      modifier.deleteDocuments(terms);
      if (!autoCommit) {
        modifier.close();
      }
      reader = IndexReader.open(dir);
      assertEquals(2, reader.numDocs());
      reader.close();

      if (autoCommit) {
        modifier.close();
      }
      dir.close();
    }
  }

  private void addDoc(IndexWriter modifier, int id, int value)
      throws IOException {
    Document doc = new Document();
    doc.add(new Field("content", "aaa", Field.Store.NO, Field.Index.TOKENIZED));
    doc.add(new Field("id", String.valueOf(id), Field.Store.YES,
        Field.Index.UN_TOKENIZED));
    doc.add(new Field("value", String.valueOf(value), Field.Store.NO,
        Field.Index.UN_TOKENIZED));
    modifier.addDocument(doc);
  }

  private int getHitCount(Directory dir, Term term) throws IOException {
    IndexSearcher searcher = new IndexSearcher(dir);
    int hitCount = searcher.search(new TermQuery(term)).length();
    searcher.close();
    return hitCount;
  }

  public void testDeletesOnDiskFull() throws IOException {
    testOperationsOnDiskFull(false);
  }

  public void testUpdatesOnDiskFull() throws IOException {
    testOperationsOnDiskFull(true);
  }

  /**
   * Make sure if modifier tries to commit but hits disk full that modifier
   * remains consistent and usable. Similar to TestIndexReader.testDiskFull().
   */
  private void testOperationsOnDiskFull(boolean updates) throws IOException {

    boolean debug = false;
    Term searchTerm = new Term("content", "aaa");
    int START_COUNT = 157;
    int END_COUNT = 144;

    for(int pass=0;pass<2;pass++) {
      boolean autoCommit = (0==pass);

      // First build up a starting index:
      RAMDirectory startDir = new RAMDirectory();
      IndexWriter writer = new IndexWriter(startDir, autoCommit,
                                           new WhitespaceAnalyzer(), true);
      for (int i = 0; i < 157; i++) {
        Document d = new Document();
        d.add(new Field("id", Integer.toString(i), Field.Store.YES,
                        Field.Index.UN_TOKENIZED));
        d.add(new Field("content", "aaa " + i, Field.Store.NO,
                        Field.Index.TOKENIZED));
        writer.addDocument(d);
      }
      writer.close();

      long diskUsage = startDir.sizeInBytes();
      long diskFree = diskUsage + 10;

      IOException err = null;

      boolean done = false;

      // Iterate w/ ever increasing free disk space:
      while (!done) {
        MockRAMDirectory dir = new MockRAMDirectory(startDir);
        IndexWriter modifier = new IndexWriter(dir, autoCommit,
                                               new WhitespaceAnalyzer());

        modifier.setMaxBufferedDocs(1000); // use flush or close
        modifier.setMaxBufferedDeleteTerms(1000); // use flush or close

        // For each disk size, first try to commit against
        // dir that will hit random IOExceptions & disk
        // full; after, give it infinite disk space & turn
        // off random IOExceptions & retry w/ same reader:
        boolean success = false;

        for (int x = 0; x < 2; x++) {

          double rate = 0.1;
          double diskRatio = ((double)diskFree) / diskUsage;
          long thisDiskFree;
          String testName;

          if (0 == x) {
            thisDiskFree = diskFree;
            if (diskRatio >= 2.0) {
              rate /= 2;
            }
            if (diskRatio >= 4.0) {
              rate /= 2;
            }