testindexreaderreopen.java

Lucene a java open-source SearchEngine Framework

package org.apache.lucene.index;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
import java.util.Set;

import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.search.Hits;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;

import junit.framework.TestCase;

public class TestIndexReaderReopen extends TestCase {
    
  public void testReopen() throws Exception {
    final Directory dir1 = new RAMDirectory();
    
    createIndex(dir1, false);
    performDefaultTests(new TestReopen() {

      protected void modifyIndex(int i) throws IOException {
        TestIndexReaderReopen.modifyIndex(i, dir1);
      }

      protected IndexReader openReader() throws IOException {
        return IndexReader.open(dir1);
      }
      
    });
    
    final Directory dir2 = new RAMDirectory();
    
    createIndex(dir2, true);
    performDefaultTests(new TestReopen() {

      protected void modifyIndex(int i) throws IOException {
        TestIndexReaderReopen.modifyIndex(i, dir2);
      }

      protected IndexReader openReader() throws IOException {
        return IndexReader.open(dir2);
      }
      
    });
  }
  
  public void testParallelReaderReopen() throws Exception {
    final Directory dir1 = new RAMDirectory();
    createIndex(dir1, true);
    final Directory dir2 = new RAMDirectory();
    createIndex(dir2, true);
    
    performDefaultTests(new TestReopen() {

      protected void modifyIndex(int i) throws IOException {
        TestIndexReaderReopen.modifyIndex(i, dir1);
        TestIndexReaderReopen.modifyIndex(i, dir2);
      }

      protected IndexReader openReader() throws IOException {
        ParallelReader pr = new ParallelReader();
        pr.add(IndexReader.open(dir1));
        pr.add(IndexReader.open(dir2));
        return pr;
      }
      
    });
    
    final Directory dir3 = new RAMDirectory();
    createIndex(dir3, true);
    final Directory dir4 = new RAMDirectory();
    createIndex(dir4, true);

    performTestsWithExceptionInReopen(new TestReopen() {

      protected void modifyIndex(int i) throws IOException {
        TestIndexReaderReopen.modifyIndex(i, dir3);
        TestIndexReaderReopen.modifyIndex(i, dir4);
      }

      protected IndexReader openReader() throws IOException {
        ParallelReader pr = new ParallelReader();
        pr.add(IndexReader.open(dir3));
        pr.add(IndexReader.open(dir4));
        pr.add(new FilterIndexReader(IndexReader.open(dir3)));
        return pr;
      }
      
    });
  }

  public void testMultiReaderReopen() throws Exception {
    final Directory dir1 = new RAMDirectory();
    createIndex(dir1, true);
    final Directory dir2 = new RAMDirectory();
    createIndex(dir2, true);

    performDefaultTests(new TestReopen() {

      protected void modifyIndex(int i) throws IOException {
        TestIndexReaderReopen.modifyIndex(i, dir1);
        TestIndexReaderReopen.modifyIndex(i, dir2);
      }

      protected IndexReader openReader() throws IOException {
        return new MultiReader(new IndexReader[] 
                        {IndexReader.open(dir1), 
                         IndexReader.open(dir2)});
      }
      
    });
    
    final Directory dir3 = new RAMDirectory();
    createIndex(dir3, true);
    final Directory dir4 = new RAMDirectory();
    createIndex(dir4, true);

    performTestsWithExceptionInReopen(new TestReopen() {

      protected void modifyIndex(int i) throws IOException {
        TestIndexReaderReopen.modifyIndex(i, dir3);
        TestIndexReaderReopen.modifyIndex(i, dir4);
      }

      protected IndexReader openReader() throws IOException {
        return new MultiReader(new IndexReader[] 
                        {IndexReader.open(dir3), 
                         IndexReader.open(dir4),
                         new FilterIndexReader(IndexReader.open(dir3))});
      }
      
    });

  }

  public void testMixedReaders() throws Exception {
    final Directory dir1 = new RAMDirectory();
    createIndex(dir1, true);
    final Directory dir2 = new RAMDirectory();
    createIndex(dir2, true);
    final Directory dir3 = new RAMDirectory();
    createIndex(dir3, false);
    final Directory dir4 = new RAMDirectory();
    createIndex(dir4, true);
    final Directory dir5 = new RAMDirectory();
    createIndex(dir5, false);
    
    performDefaultTests(new TestReopen() {

      protected void modifyIndex(int i) throws IOException {
        // only change norms in this index to maintain the same number of docs for each of ParallelReader's subreaders
        if (i == 1) TestIndexReaderReopen.modifyIndex(i, dir1);  
        
        TestIndexReaderReopen.modifyIndex(i, dir4);
        TestIndexReaderReopen.modifyIndex(i, dir5);
      }

      protected IndexReader openReader() throws IOException {
        ParallelReader pr = new ParallelReader();
        pr.add(IndexReader.open(dir1));
        pr.add(IndexReader.open(dir2));
        MultiReader mr = new MultiReader(new IndexReader[] {
            IndexReader.open(dir3), IndexReader.open(dir4)});
        return new MultiReader(new IndexReader[] {
           pr, mr, IndexReader.open(dir5)});
      }
    });
  }  

  
  private void performDefaultTests(TestReopen test) throws Exception {
    IndexReader index1 = test.openReader();
    IndexReader index2 = test.openReader();
        
    TestIndexReader.assertIndexEquals(index1, index2);
    
    // verify that reopen() does not return a new reader instance
    // in case the index has no changes
    ReaderCouple couple = refreshReader(index2, false);
    assertTrue(couple.refreshedReader == index2);
    
    couple = refreshReader(index2, test, 0, true);
    index1 = couple.newReader;
    IndexReader index2_refreshed = couple.refreshedReader;
    index2.close();
    
    // test if refreshed reader and newly opened reader return equal results
    TestIndexReader.assertIndexEquals(index1, index2_refreshed);
    
    index1.close();
    index2_refreshed.close();
    assertReaderClosed(index2, true, true);
    assertReaderClosed(index2_refreshed, true, true);
    
    index2 = test.openReader();
    
    for (int i = 1; i < 4; i++) {
      
      index1.close();
      couple = refreshReader(index2, test, i, true);
      // refresh IndexReader
      index2.close();
      
      index2 = couple.refreshedReader;
      index1 = couple.newReader;
      TestIndexReader.assertIndexEquals(index1, index2);
    }
    
    index1.close();
    index2.close();
    assertReaderClosed(index1, true, true);
    assertReaderClosed(index2, true, true);
  }
  
  public void testReferenceCounting() throws IOException {
   
    for (int mode = 0; mode < 4; mode++) {
      Directory dir1 = new RAMDirectory();
      createIndex(dir1, true);
     
      IndexReader reader0 = IndexReader.open(dir1);
      assertRefCountEquals(1, reader0);

      assertTrue(reader0 instanceof MultiSegmentReader);
      SegmentReader[] subReaders0 = ((MultiSegmentReader) reader0).getSubReaders();
      for (int i = 0; i < subReaders0.length; i++) {
        assertRefCountEquals(1, subReaders0[i]);
      }
      
      // delete first document, so that only one of the subReaders have to be re-opened
      IndexReader modifier = IndexReader.open(dir1);
      modifier.deleteDocument(0);
      modifier.close();
      
      IndexReader reader1 = refreshReader(reader0, true).refreshedReader;
      assertTrue(reader1 instanceof MultiSegmentReader);
      SegmentReader[] subReaders1 = ((MultiSegmentReader) reader1).getSubReaders();
      assertEquals(subReaders0.length, subReaders1.length);
      
      for (int i = 0; i < subReaders0.length; i++) {
        assertRefCountEquals(2, subReaders0[i]);
        if (subReaders0[i] != subReaders1[i]) {
          assertRefCountEquals(1, subReaders1[i]);
        }
      }

      // delete first document, so that only one of the subReaders have to be re-opened
      modifier = IndexReader.open(dir1);
      modifier.deleteDocument(1);
      modifier.close();

      IndexReader reader2 = refreshReader(reader1, true).refreshedReader;
      assertTrue(reader2 instanceof MultiSegmentReader);
      SegmentReader[] subReaders2 = ((MultiSegmentReader) reader2).getSubReaders();
      assertEquals(subReaders1.length, subReaders2.length);
      
      for (int i = 0; i < subReaders2.length; i++) {
        if (subReaders2[i] == subReaders1[i]) {
          if (subReaders1[i] == subReaders0[i]) {
            assertRefCountEquals(3, subReaders2[i]);
          } else {
            assertRefCountEquals(2, subReaders2[i]);
          }
        } else {
          assertRefCountEquals(1, subReaders2[i]);
          if (subReaders0[i] == subReaders1[i]) {
            assertRefCountEquals(3, subReaders2[i]);
            assertRefCountEquals(2, subReaders0[i]);
          } else {
            assertRefCountEquals(3, subReaders0[i]);
            assertRefCountEquals(1, subReaders1[i]);
          }
        }
      }
      
      IndexReader reader3 = refreshReader(reader0, true).refreshedReader;
      assertTrue(reader3 instanceof MultiSegmentReader);
      SegmentReader[] subReaders3 = ((MultiSegmentReader) reader3).getSubReaders();
      assertEquals(subReaders3.length, subReaders0.length);
      
      // try some permutations
      switch (mode) {
      case 0:
        reader0.close();
        reader1.close();
        reader2.close();
        reader3.close();
        break;
      case 1:
        reader3.close();
        reader2.close();
        reader1.close();
        reader0.close();
        break;
      case 2:
        reader2.close();
        reader3.close();
        reader0.close();
        reader1.close();
        break;
      case 3:
        reader1.close();
        reader3.close();
        reader2.close();
        reader0.close();
        break;
      }      
      
      assertReaderClosed(reader0, true, true);
      assertReaderClosed(reader1, true, true);
      assertReaderClosed(reader2, true, true);
      assertReaderClosed(reader3, true, true);
    }
  }


  public void testReferenceCountingMultiReader() throws IOException {
    for (int mode = 0; mode <=1; mode++) {
      Directory dir1 = new RAMDirectory();
      createIndex(dir1, false);
      Directory dir2 = new RAMDirectory();
      createIndex(dir2, true);
      
      IndexReader reader1 = IndexReader.open(dir1);
      assertRefCountEquals(1, reader1);
      
      IndexReader multiReader1 = new MultiReader(new IndexReader[] {reader1, IndexReader.open(dir2)}, (mode == 0));
      modifyIndex(0, dir2);
      assertRefCountEquals(1 + mode, reader1);
      
      IndexReader multiReader2 = multiReader1.reopen();
      // index1 hasn't changed, so multiReader2 should share reader1 now with multiReader1
      assertRefCountEquals(2 + mode, reader1);
      
      modifyIndex(0, dir1);
      IndexReader reader2 = reader1.reopen();
      assertRefCountEquals(3 + mode, reader1);
      
      modifyIndex(1, dir1);
      IndexReader reader3 = reader2.reopen();
      assertRefCountEquals(4 + mode, reader1);
      assertRefCountEquals(1, reader2);
      
      multiReader1.close();
      assertRefCountEquals(3 + mode, reader1);
      
      multiReader1.close();
      assertRefCountEquals(3 + mode, reader1);
      
      reader1.close();
      assertRefCountEquals(3, reader1);
      
      multiReader2.close();
      assertRefCountEquals(2, reader1);
      
      multiReader2.close();
      assertRefCountEquals(2, reader1);
      
      reader3.close();
      assertRefCountEquals(1, reader1);
      assertReaderOpen(reader1);
      
      reader2.close();
      assertRefCountEquals(0, reader1);
      assertReaderClosed(reader1, true, false);
      
      reader2.close();
      assertRefCountEquals(0, reader1);
      
      reader3.close();
      assertRefCountEquals(0, reader1);
      assertReaderClosed(reader1, true, true);
    }

  }

  public void testReferenceCountingParallelReader() throws IOException {
    for (int mode = 0; mode <=1; mode++) {
      Directory dir1 = new RAMDirectory();
      createIndex(dir1, false);
      Directory dir2 = new RAMDirectory();
      createIndex(dir2, true);
      
      IndexReader reader1 = IndexReader.open(dir1);
      assertRefCountEquals(1, reader1);
      
      ParallelReader parallelReader1 = new ParallelReader(mode == 0);
      parallelReader1.add(reader1);
      parallelReader1.add(IndexReader.open(dir2));
      modifyIndex(1, dir2);
      assertRefCountEquals(1 + mode, reader1);
      
      IndexReader parallelReader2 = parallelReader1.reopen();
      // index1 hasn't changed, so parallelReader2 should share reader1 now with multiReader1
      assertRefCountEquals(2 + mode, reader1);
      
      modifyIndex(0, dir1);
      modifyIndex(0, dir2);
      IndexReader reader2 = reader1.reopen();
      assertRefCountEquals(3 + mode, reader1);
      
      modifyIndex(4, dir1);
      IndexReader reader3 = reader2.reopen();
      assertRefCountEquals(4 + mode, reader1);
      assertRefCountEquals(1, reader2);
      
      parallelReader1.close();
      assertRefCountEquals(3 + mode, reader1);
      
      parallelReader1.close();
      assertRefCountEquals(3 + mode, reader1);
      
      reader1.close();
      assertRefCountEquals(3, reader1);
      
      parallelReader2.close();
      assertRefCountEquals(2, reader1);
      
      parallelReader2.close();
      assertRefCountEquals(2, reader1);
      
      reader3.close();
      assertRefCountEquals(1, reader1);
      assertReaderOpen(reader1);
      
      reader2.close();
      assertRefCountEquals(0, reader1);
      assertReaderClosed(reader1, true, false);
      
      reader2.close();