Examples of MockDirectoryWrapper

org.apache.lucene.store.MockDirectoryWrapper
This is a Directory Wrapper that adds methods intended to be used only by unit tests. It also adds a number of features useful for testing:
- Instances created by {@link LuceneTestCase#newDirectory()} are tracked to ensure they are closed by the test.
- When a MockDirectoryWrapper is closed, it will throw an exception if it has any open files against it (with a stacktrace indicating where they were opened from).
- When a MockDirectoryWrapper is closed, it runs CheckIndex to test if the index was corrupted.
- MockDirectoryWrapper simulates some "features" of Windows, such as refusing to write/delete to open files.

Examples of org.apache.lucene.store.MockDirectoryWrapper

import org.apache.lucene.util._TestUtil;


public class TestPerSegmentDeletes extends LuceneTestCase {
  public void testDeletes1() throws Exception {
    //IndexWriter.debug2 = System.out;
    Directory dir = new MockDirectoryWrapper(new Random(random().nextLong()), new RAMDirectory());
    IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT,
        new MockAnalyzer(random()));
    iwc.setMergeScheduler(new SerialMergeScheduler());
    iwc.setMaxBufferedDocs(5000);
    iwc.setRAMBufferSizeMB(100);
    RangeMergePolicy fsmp = new RangeMergePolicy(false);
    iwc.setMergePolicy(fsmp);
    IndexWriter writer = new IndexWriter(dir, iwc);
    for (int x = 0; x < 5; x++) {
      writer.addDocument(DocHelper.createDocument(x, "1", 2));
      //System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
    }
    //System.out.println("commit1");
    writer.commit();
    assertEquals(1, writer.segmentInfos.size());
    for (int x = 5; x < 10; x++) {
      writer.addDocument(DocHelper.createDocument(x, "2", 2));
      //System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
    }
    //System.out.println("commit2");
    writer.commit();
    assertEquals(2, writer.segmentInfos.size());


    for (int x = 10; x < 15; x++) {
      writer.addDocument(DocHelper.createDocument(x, "3", 2));
      //System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
    }


    writer.deleteDocuments(new Term("id", "1"));


    writer.deleteDocuments(new Term("id", "11"));


    // flushing without applying deletes means
    // there will still be deletes in the segment infos
    writer.flush(false, false);
    assertTrue(writer.bufferedUpdatesStream.any());


    // get reader flushes pending deletes
    // so there should not be anymore
    IndexReader r1 = writer.getReader();
    assertFalse(writer.bufferedUpdatesStream.any());
    r1.close();


    // delete id:2 from the first segment
    // merge segments 0 and 1
    // which should apply the delete id:2
    writer.deleteDocuments(new Term("id", "2"));
    writer.flush(false, false);
    fsmp = (RangeMergePolicy) writer.getConfig().getMergePolicy();
    fsmp.doMerge = true;
    fsmp.start = 0;
    fsmp.length = 2;
    writer.maybeMerge();


    assertEquals(2, writer.segmentInfos.size());


    // id:2 shouldn't exist anymore because
    // it's been applied in the merge and now it's gone
    IndexReader r2 = writer.getReader();
    int[] id2docs = toDocsArray(new Term("id", "2"), null, r2);
    assertTrue(id2docs == null);
    r2.close();


    /**
    // added docs are in the ram buffer
    for (int x = 15; x < 20; x++) {
      writer.addDocument(TestIndexWriterReader.createDocument(x, "4", 2));
      System.out.println("numRamDocs(" + x + ")" + writer.numRamDocs());
    }
    assertTrue(writer.numRamDocs() > 0);
    // delete from the ram buffer
    writer.deleteDocuments(new Term("id", Integer.toString(13)));


    Term id3 = new Term("id", Integer.toString(3));


    // delete from the 1st segment
    writer.deleteDocuments(id3);


    assertTrue(writer.numRamDocs() > 0);


    //System.out
    //    .println("segdels1:" + writer.docWriter.deletesToString());


    //assertTrue(writer.docWriter.segmentDeletes.size() > 0);


    // we cause a merge to happen
    fsmp.doMerge = true;
    fsmp.start = 0;
    fsmp.length = 2;
    System.out.println("maybeMerge "+writer.segmentInfos);


    SegmentInfo info0 = writer.segmentInfos.info(0);
    SegmentInfo info1 = writer.segmentInfos.info(1);


    writer.maybeMerge();
    System.out.println("maybeMerge after "+writer.segmentInfos);
    // there should be docs in RAM
    assertTrue(writer.numRamDocs() > 0);


    // assert we've merged the 1 and 2 segments
    // and still have a segment leftover == 2
    assertEquals(2, writer.segmentInfos.size());
    assertFalse(segThere(info0, writer.segmentInfos));
    assertFalse(segThere(info1, writer.segmentInfos));


    //System.out.println("segdels2:" + writer.docWriter.deletesToString());


    //assertTrue(writer.docWriter.segmentDeletes.size() > 0);


    IndexReader r = writer.getReader();
    IndexReader r1 = r.getSequentialSubReaders()[0];
    printDelDocs(r1.getLiveDocs());
    int[] docs = toDocsArray(id3, null, r);
    System.out.println("id3 docs:"+Arrays.toString(docs));
    // there shouldn't be any docs for id:3
    assertTrue(docs == null);
    r.close();


    part2(writer, fsmp);
    **/
    // System.out.println("segdels2:"+writer.docWriter.segmentDeletes.toString());
    //System.out.println("close");
    writer.close();
    dir.close();
  }

View Full Code Here

Examples of org.apache.lucene.store.MockDirectoryWrapper

    
    IndexerThreadInterrupt() throws IOException {
      this.random = new Random(random().nextLong());
      // make a little directory for addIndexes
      // LUCENE-2239: won't work with NIOFS/MMAP
      adder = new MockDirectoryWrapper(random, new RAMDirectory());
      IndexWriterConfig conf = newIndexWriterConfig(random,
          TEST_VERSION_CURRENT, new MockAnalyzer(random));
      IndexWriter w = new IndexWriter(adder, conf);
      Document doc = new Document();
      doc.add(newStringField(random, "id", "500", Field.Store.NO));

View Full Code Here

Examples of org.apache.lucene.store.MockDirectoryWrapper

    }


    @Override
    public void run() {
      // LUCENE-2239: won't work with NIOFS/MMAP
      MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory());


      // When interrupt arrives in w.close(), when it's
      // writing liveDocs, this can lead to double-write of
      // _X_N.del:
      //dir.setPreventDoubleWrite(false);

View Full Code Here

Examples of org.apache.lucene.store.MockDirectoryWrapper

    dir.close();
  }


  public void testDeleteAllNRTLeftoverFiles() throws Exception {


    Directory d = new MockDirectoryWrapper(random(), new RAMDirectory());
    IndexWriter w = new IndexWriter(d, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
    Document doc = new Document();
    for(int i = 0; i < 20; i++) {
      for(int j = 0; j < 100; ++j) {
        w.addDocument(doc);
      }
      w.commit();
      DirectoryReader.open(w, true).close();


      w.deleteAll();
      w.commit();
      // Make sure we accumulate no files except for empty
      // segments_N and segments.gen:
      assertTrue(d.listAll().length <= 2);
    }


    w.close();
    d.close();
  }

View Full Code Here

Examples of org.apache.lucene.store.MockDirectoryWrapper

    w.close();
    d.close();
  }


  public void testNRTReaderVersion() throws Exception {
    Directory d = new MockDirectoryWrapper(random(), new RAMDirectory());
    IndexWriter w = new IndexWriter(d, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())));
    Document doc = new Document();
    doc.add(newStringField("id", "0", Field.Store.YES));
    w.addDocument(doc);
    DirectoryReader r = w.getReader();
    long version = r.getVersion();
    r.close();


    w.addDocument(doc);
    r = w.getReader();
    long version2 = r.getVersion();
    r.close();
    assert(version2 > version);


    w.deleteDocuments(new Term("id", "0"));
    r = w.getReader();
    w.close();
    long version3 = r.getVersion();
    r.close();
    assert(version3 > version2);
    d.close();
  }

View Full Code Here

Examples of org.apache.lucene.store.MockDirectoryWrapper

    dir.close();
  }
  
  // LUCENE-1647
  public void testIndexReaderUnDeleteAll() throws Exception {
    MockDirectoryWrapper dir = newDirectory();
    dir.setPreventDoubleWrite(false);
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
        TEST_VERSION_CURRENT, new MockAnalyzer(random)));
    writer.addDocument(createDocument("a"));
    writer.addDocument(createDocument("b"));
    writer.addDocument(createDocument("c"));
    writer.close();
    IndexReader reader = IndexReader.open(dir, false);
    reader.deleteDocuments(new Term("id", "a"));
    reader.flush();
    reader.deleteDocuments(new Term("id", "b"));
    reader.undeleteAll();
    reader.deleteDocuments(new Term("id", "b"));
    reader.close();
    IndexReader.open(dir,true).close();
    dir.close();
  }

View Full Code Here

Examples of org.apache.lucene.store.MockDirectoryWrapper

  private String[] docs;
  private String fieldName;
  
  public SingleFieldTestDb(Random random, String[] documents, String fName) {
    try {
      db = new MockDirectoryWrapper(random, new RAMDirectory());
      docs = documents;
      fieldName = fName;
      IndexWriter writer = new IndexWriter(db, new IndexWriterConfig(
          Version.LUCENE_CURRENT,
          new MockAnalyzer(random)));

View Full Code Here

Examples of org.apache.lucene.store.MockDirectoryWrapper

   * closing the writer, call abort and verify reader sees
   * nothing was added.  Then verify we can open the index
   * and add docs to it.
   */
  public void testCommitOnCloseAbort() throws IOException {
    MockDirectoryWrapper dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(10));
    for (int i = 0; i < 14; i++) {
      TestIndexWriter.addDoc(writer);
    }
    writer.close();


    Term searchTerm = new Term("content", "aaa");
    IndexSearcher searcher = new IndexSearcher(dir, false);
    ScoreDoc[] hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
    assertEquals("first number of hits", 14, hits.length);
    searcher.close();


    writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
      .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10));
    for(int j=0;j<17;j++) {
      TestIndexWriter.addDoc(writer);
    }
    // Delete all docs:
    writer.deleteDocuments(searchTerm);


    searcher = new IndexSearcher(dir, false);
    hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
    assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
    searcher.close();


    // Now, close the writer:
    writer.rollback();


    TestIndexWriter.assertNoUnreferencedFiles(dir, "unreferenced files remain after rollback()");


    searcher = new IndexSearcher(dir, false);
    hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
    assertEquals("saw changes after writer.abort", 14, hits.length);
    searcher.close();


    // Now make sure we can re-open the index, add docs,
    // and all is good:
    writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
      .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10));


    // On abort, writer in fact may write to the same
    // segments_N file:
    dir.setPreventDoubleWrite(false);


    for(int i=0;i<12;i++) {
      for(int j=0;j<17;j++) {
        TestIndexWriter.addDoc(writer);
      }
      searcher = new IndexSearcher(dir, false);
      hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
      assertEquals("reader incorrectly sees changes from writer", 14, hits.length);
      searcher.close();
    }


    writer.close();
    searcher = new IndexSearcher(dir, false);
    hits = searcher.search(new TermQuery(searchTerm), null, 1000).scoreDocs;
    assertEquals("didn't see changes after close", 218, hits.length);
    searcher.close();


    dir.close();
  }

View Full Code Here

Examples of org.apache.lucene.store.MockDirectoryWrapper

   * that are not referenced by the starting segments
   * file.  We check this by using MockDirectoryWrapper to
   * measure max temp disk space used.
   */
  public void testCommitOnCloseDiskUsage() throws IOException {
    MockDirectoryWrapper dir = newDirectory();
    Analyzer analyzer;
    if (random.nextBoolean()) {
      // no payloads
     analyzer = new Analyzer() {
        @Override
        public TokenStream tokenStream(String fieldName, Reader reader) {
          return new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
        }
      };
    } else {
      // fixed length payloads
      final int length = random.nextInt(200);
      analyzer = new Analyzer() {
        @Override
        public TokenStream tokenStream(String fieldName, Reader reader) {
          return new MockFixedLengthPayloadFilter(random,
              new MockTokenizer(reader, MockTokenizer.WHITESPACE, true),
              length);
        }
      };
    }
    
    IndexWriter writer  = new IndexWriter(
        dir,
        newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).
            setMaxBufferedDocs(10).
            setReaderPooling(false).
            setMergePolicy(newLogMergePolicy(10))
    );
    for(int j=0;j<30;j++) {
      TestIndexWriter.addDocWithIndex(writer, j);
    }
    writer.close();
    dir.resetMaxUsedSizeInBytes();


    dir.setTrackDiskUsage(true);
    long startDiskUsage = dir.getMaxUsedSizeInBytes();
    writer = new IndexWriter(
        dir,
        newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)
            .setOpenMode(OpenMode.APPEND).
            setMaxBufferedDocs(10).
            setMergeScheduler(new SerialMergeScheduler()).
            setReaderPooling(false).
            setMergePolicy(newLogMergePolicy(10))


    );
    for(int j=0;j<1470;j++) {
      TestIndexWriter.addDocWithIndex(writer, j);
    }
    long midDiskUsage = dir.getMaxUsedSizeInBytes();
    dir.resetMaxUsedSizeInBytes();
    writer.optimize();
    writer.close();


    IndexReader.open(dir, true).close();


    long endDiskUsage = dir.getMaxUsedSizeInBytes();


    // Ending index is 50X as large as starting index; due
    // to 3X disk usage normally we allow 150X max
    // transient usage.  If something is wrong w/ deleter
    // and it doesn't delete intermediate segments then it
    // will exceed this 150X:
    // System.out.println("start " + startDiskUsage + "; mid " + midDiskUsage + ";end " + endDiskUsage);
    assertTrue("writer used too much space while adding documents: mid=" + midDiskUsage + " start=" + startDiskUsage + " end=" + endDiskUsage + " max=" + (startDiskUsage*150),
               midDiskUsage < 150*startDiskUsage);
    assertTrue("writer used too much space after close: endDiskUsage=" + endDiskUsage + " startDiskUsage=" + startDiskUsage + " max=" + (startDiskUsage*150),
               endDiskUsage < 150*startDiskUsage);
    dir.close();
  }

View Full Code Here

Examples of org.apache.lucene.store.MockDirectoryWrapper

   * Verify that calling optimize when writer is open for
   * "commit on close" works correctly both for rollback()
   * and close().
   */
  public void testCommitOnCloseOptimize() throws IOException {
    MockDirectoryWrapper dir = newDirectory();
    // Must disable throwing exc on double-write: this
    // test uses IW.rollback which easily results in
    // writing to same file more than once
    dir.setPreventDoubleWrite(false);
    IndexWriter writer = new IndexWriter(
        dir,
        newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
            setMaxBufferedDocs(10).
            setMergePolicy(newLogMergePolicy(10))
    );
    for(int j=0;j<17;j++) {
      TestIndexWriter.addDocWithIndex(writer, j);
    }
    writer.close();


    writer  = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
    writer.optimize();


    if (VERBOSE) {
      writer.setInfoStream(System.out);
    }


    // Open a reader before closing (commiting) the writer:
    IndexReader reader = IndexReader.open(dir, true);


    // Reader should see index as unoptimized at this
    // point:
    assertFalse("Reader incorrectly sees that the index is optimized", reader.isOptimized());
    reader.close();


    // Abort the writer:
    writer.rollback();
    TestIndexWriter.assertNoUnreferencedFiles(dir, "aborted writer after optimize");


    // Open a reader after aborting writer:
    reader = IndexReader.open(dir, true);


    // Reader should still see index as unoptimized:
    assertFalse("Reader incorrectly sees that the index is optimized", reader.isOptimized());
    reader.close();


    if (VERBOSE) {
      System.out.println("TEST: do real optimize");
    }
    writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
    if (VERBOSE) {
      writer.setInfoStream(System.out);
    }
    writer.optimize();
    writer.close();


    if (VERBOSE) {
      System.out.println("TEST: writer closed");
    }
    TestIndexWriter.assertNoUnreferencedFiles(dir, "aborted writer after optimize");


    // Open a reader after aborting writer:
    reader = IndexReader.open(dir, true);


    // Reader should still see index as unoptimized:
    assertTrue("Reader incorrectly sees that the index is unoptimized", reader.isOptimized());
    reader.close();
    dir.close();
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.