testindexreaderreopen.java

Lucene a java open-source SearchEngine Framework

      assertRefCountEquals(0, reader1);
      
      reader3.close();
      assertRefCountEquals(0, reader1);
      assertReaderClosed(reader1, true, true);
    }

  }
  
  public void testNormsRefCounting() throws IOException {
    Directory dir1 = new RAMDirectory();
    createIndex(dir1, false);
    
    SegmentReader reader1 = (SegmentReader) IndexReader.open(dir1);
    IndexReader modifier = IndexReader.open(dir1);
    modifier.deleteDocument(0);
    modifier.close();
    
    SegmentReader reader2 = (SegmentReader) reader1.reopen();
    modifier = IndexReader.open(dir1);
    modifier.setNorm(1, "field1", 50);
    modifier.setNorm(1, "field2", 50);
    modifier.close();
    
    SegmentReader reader3 = (SegmentReader) reader2.reopen();
    modifier = IndexReader.open(dir1);
    modifier.deleteDocument(2);
    modifier.close();
    SegmentReader reader4 = (SegmentReader) reader3.reopen();

    modifier = IndexReader.open(dir1);
    modifier.deleteDocument(3);
    modifier.close();
    SegmentReader reader5 = (SegmentReader) reader3.reopen();
    
    // Now reader2-reader5 references reader1. reader1 and reader2
    // share the same norms. reader3, reader4, reader5 also share norms.
    assertRefCountEquals(5, reader1);
    assertFalse(reader1.normsClosed());
    reader1.close();
    assertRefCountEquals(4, reader1);
    assertFalse(reader1.normsClosed());
    reader2.close();
    assertRefCountEquals(3, reader1);
    // now the norms for field1 and field2 should be closed
    assertTrue(reader1.normsClosed("field1"));
    assertTrue(reader1.normsClosed("field2"));
    // but the norms for field3 and field4 should still be open
    assertFalse(reader1.normsClosed("field3"));
    assertFalse(reader1.normsClosed("field4"));
    
    reader3.close();
    assertRefCountEquals(2, reader1);
    assertFalse(reader3.normsClosed());
    reader5.close();
    assertRefCountEquals(1, reader1);
    assertFalse(reader3.normsClosed());
    reader4.close();
    assertRefCountEquals(0, reader1);
    
    // and now all norms that reader1 used should be closed
    assertTrue(reader1.normsClosed());
    
    // now that reader3, reader4 and reader5 are closed,
    // the norms that those three readers shared should be
    // closed as well
    assertTrue(reader3.normsClosed());
  }
  
  private void performTestsWithExceptionInReopen(TestReopen test) throws Exception {
    IndexReader index1 = test.openReader();
    IndexReader index2 = test.openReader();

    TestIndexReader.assertIndexEquals(index1, index2);
    
    try {
      ReaderCouple couple = refreshReader(index1, test, 0, true);
      fail("Expected exception not thrown.");
    } catch (Exception e) {
      // expected exception
    }
    
    // index2 should still be usable and unaffected by the failed reopen() call
    TestIndexReader.assertIndexEquals(index1, index2);
  }
  
  public void testThreadSafety() throws Exception {
    final Directory dir = new RAMDirectory();
    final int n = 150;

    IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer());
    for (int i = 0; i < n; i++) {
      writer.addDocument(createDocument(i, 3));
    }
    writer.optimize();
    writer.close();

    final TestReopen test = new TestReopen() {      
      protected void modifyIndex(int i) throws IOException {
        if (i % 3 == 0) {
          IndexReader modifier = IndexReader.open(dir);
          modifier.setNorm(i, "field1", 50);
          modifier.close();
        } else if (i % 3 == 1) {
          IndexReader modifier = IndexReader.open(dir);
          modifier.deleteDocument(i);
          modifier.close();
        } else {
          IndexWriter modifier = new IndexWriter(dir, new StandardAnalyzer());
          modifier.addDocument(createDocument(n + i, 6));
          modifier.close();
        }
      }

      protected IndexReader openReader() throws IOException {
        return IndexReader.open(dir);
      }      
    };
    
    final List readers = Collections.synchronizedList(new ArrayList());
    IndexReader firstReader = IndexReader.open(dir);
    IndexReader reader = firstReader;
    final Random rnd = new Random();
    
    ReaderThread[] threads = new ReaderThread[n];
    final Set readersToClose = Collections.synchronizedSet(new HashSet());
    
    for (int i = 0; i < n; i++) {
      if (i % 10 == 0) {
        IndexReader refreshed = reader.reopen();
        if (refreshed != reader) {
          readersToClose.add(reader);
        }
        reader = refreshed;
      }
      final IndexReader r = reader;
      
      final int index = i;    
      
      ReaderThreadTask task;
      
      if (i < 20 ||( i >=50 && i < 70) || i > 90) {
        task = new ReaderThreadTask() {
          
          public void run() throws Exception {
            while (!stopped) {
              if (index % 2 == 0) {
                // refresh reader synchronized
                ReaderCouple c = (refreshReader(r, test, index, true));
                readersToClose.add(c.newReader);
                readersToClose.add(c.refreshedReader);
                readers.add(c);
                // prevent too many readers
                break;
              } else {
                // not synchronized
                IndexReader refreshed = r.reopen();
                
                
                IndexSearcher searcher = new IndexSearcher(refreshed);
                Hits hits = searcher.search(new TermQuery(new Term("field1", "a" + rnd.nextInt(refreshed.maxDoc()))));
                if (hits.length() > 0) {
                  hits.doc(0);
                }
                
                // r might have changed because this is not a 
                // synchronized method. However we don't want
                // to make it synchronized to test 
                // thread-safety of IndexReader.close().
                // That's why we add refreshed also to 
                // readersToClose, because double closing is fine
                if (refreshed != r) {
                  refreshed.close();
                }
                readersToClose.add(refreshed);
              }
              try {
                synchronized(this) {
                  wait(1000);
                }
              } catch (InterruptedException e) {}
            }
          }
          
        };
      } else {
        task = new ReaderThreadTask() {
          public void run() throws Exception {
            while (!stopped) {
              int numReaders = readers.size();
              if (numReaders > 0) {
                ReaderCouple c = (ReaderCouple) readers.get(rnd.nextInt(numReaders));
                TestIndexReader.assertIndexEquals(c.newReader, c.refreshedReader);
              }
              
              try {
                synchronized(this) {
                  wait(100);
                }
              } catch (InterruptedException e) {}
            }
                        
          }
          
        };
      }
      
      threads[i] = new ReaderThread(task);
      threads[i].start();
    }
    
    synchronized(this) {
      try {
        wait(15000);
      } catch(InterruptedException e) {}
    }
    
    for (int i = 0; i < n; i++) {
      if (threads[i] != null) {
        threads[i].stopThread();
      }
    }
    
    for (int i = 0; i < n; i++) {
      if (threads[i] != null) {
        try {
          threads[i].join();
          if (threads[i].exception != null) {
            throw threads[i].exception;
          }
        } catch (InterruptedException e) {}
      }
      
    }
    
    Iterator it = readersToClose.iterator();
    while (it.hasNext()) {
      ((IndexReader) it.next()).close();
    }
    
    firstReader.close();
    reader.close();
    
    it = readersToClose.iterator();
    while (it.hasNext()) {
      assertReaderClosed((IndexReader) it.next(), true, true);
    }

    assertReaderClosed(reader, true, true);
    assertReaderClosed(firstReader, true, true);
  }
  
  private static class ReaderCouple {
    ReaderCouple(IndexReader r1, IndexReader r2) {
      newReader = r1;
      refreshedReader = r2;
    }
    
    IndexReader newReader;
    IndexReader refreshedReader;
  }
  
  private abstract static class ReaderThreadTask {
    protected boolean stopped;
    public void stop() {
      this.stopped = true;
    }
    
    public abstract void run() throws Exception;
  }
  
  private static class ReaderThread extends Thread {
    private ReaderThreadTask task;
    private Exception exception;
    
    
    ReaderThread(ReaderThreadTask task) {
      this.task = task;
    }
    
    public void stopThread() {
      this.task.stop();
    }
    
    public void run() {
      try {
        this.task.run();
      } catch (Exception e) {
        this.exception = e;
      }
    }
  }
  
  private Object createReaderMutex = new Object();
  
  private ReaderCouple refreshReader(IndexReader reader, boolean hasChanges) throws IOException {
    return refreshReader(reader, null, -1, hasChanges);
  }
  
  private ReaderCouple refreshReader(IndexReader reader, TestReopen test, int modify, boolean hasChanges) throws IOException {
    synchronized (createReaderMutex) {
      IndexReader r = null;
      if (test != null) {
        test.modifyIndex(modify);
        r = test.openReader();
      }
      
      IndexReader refreshed = reader.reopen();
      if (hasChanges) {
        if (refreshed == reader) {
          fail("No new IndexReader instance created during refresh.");
        }
      } else {
        if (refreshed != reader) {
          fail("New IndexReader instance created during refresh even though index had no changes.");
        }
      }
      
      return new ReaderCouple(r, refreshed);
    }
  }
  
  private static void createIndex(Directory dir, boolean multiSegment) throws IOException {
    IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer());
    
    w.setMergePolicy(new LogDocMergePolicy());
    
    for (int i = 0; i < 100; i++) {
      w.addDocument(createDocument(i, 4));
      if (multiSegment && (i % 10) == 0) {
        w.flush();
      }
    }
    
    if (!multiSegment) {
      w.optimize();
    }
    
    w.close();
    
    IndexReader r = IndexReader.open(dir);
    if (multiSegment) {
      assertTrue(r instanceof MultiSegmentReader);
    } else {
      assertTrue(r instanceof SegmentReader);
    }
    r.close();
  }

  private static Document createDocument(int n, int numFields) {
    StringBuffer sb = new StringBuffer();
    Document doc = new Document();
    sb.append("a");
    sb.append(n);
    doc.add(new Field("field1", sb.toString(), Store.YES, Index.TOKENIZED));
    sb.append(" b");
    sb.append(n);
    for (int i = 1; i < numFields; i++) {
      doc.add(new Field("field" + (i+1), sb.toString(), Store.YES, Index.TOKENIZED));
    }
    return doc;
  }

  private static void modifyIndex(int i, Directory dir) throws IOException {
    switch (i) {
      case 0: {
        IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer());
        w.deleteDocuments(new Term("field2", "a11"));
        w.deleteDocuments(new Term("field2", "b30"));
        w.close();
        break;
      }
      case 1: {
        IndexReader reader = IndexReader.open(dir);
        reader.setNorm(4, "field1", 123);
        reader.setNorm(44, "field2", 222);
        reader.setNorm(44, "field4", 22);
        reader.close();
        break;
      }
      case 2: {
        IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer());
        w.optimize();
        w.close();
        break;
      }
      case 3: {
        IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer());
        w.addDocument(createDocument(101, 4));
        w.optimize();
        w.addDocument(createDocument(102, 4));
        w.addDocument(createDocument(103, 4));
        w.close();
        break;
      }
      case 4: {
        IndexReader reader = IndexReader.open(dir);
        reader.setNorm(5, "field1", 123);
        reader.setNorm(55, "field2", 222);
        reader.close();
        break;
      }

    }
  }  
  
  private void assertReaderClosed(IndexReader reader, boolean checkSubReaders, boolean checkNormsClosed) {
    assertEquals(0, reader.getRefCount());
    
    if (checkNormsClosed && reader instanceof SegmentReader) {
      assertTrue(((SegmentReader) reader).normsClosed());
    }
    
    if (checkSubReaders) {
      if (reader instanceof MultiSegmentReader) {
        SegmentReader[] subReaders = ((MultiSegmentReader) reader).getSubReaders();
        for (int i = 0; i < subReaders.length; i++) {
          assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed);
        }
      }
      
      if (reader instanceof MultiReader) {
        IndexReader[] subReaders = ((MultiReader) reader).getSubReaders();
        for (int i = 0; i < subReaders.length; i++) {
          assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed);
        }
      }
      
      if (reader instanceof ParallelReader) {
        IndexReader[] subReaders = ((ParallelReader) reader).getSubReaders();
        for (int i = 0; i < subReaders.length; i++) {
          assertReaderClosed(subReaders[i], checkSubReaders, checkNormsClosed);
        }
      }
    }
  }

  private void assertReaderOpen(IndexReader reader) {
    reader.ensureOpen();
    
    if (reader instanceof MultiSegmentReader) {
      SegmentReader[] subReaders = ((MultiSegmentReader) reader).getSubReaders();
      for (int i = 0; i < subReaders.length; i++) {
        assertReaderOpen(subReaders[i]);
      }
    }
  }

  private void assertRefCountEquals(int refCount, IndexReader reader) {
    assertEquals("Reader has wrong refCount value.", refCount, reader.getRefCount());
  }


  private abstract static class TestReopen {
    protected abstract IndexReader openReader() throws IOException;
    protected abstract void modifyIndex(int i) throws IOException;
  }
  
}