Examples of org.apache.lucene.index.SnapshotDeletionPolicy

org.apache.lucene.index.SnapshotDeletionPolicy
A {@link IndexDeletionPolicy} that wraps around any other{@link IndexDeletionPolicy} and adds the ability to hold andlater release a single "snapshot" of an index. While the snapshot is held, the {@link IndexWriter} will notremove any files associated with it even if the index is otherwise being actively, arbitrarily changed. Because we wrap another arbitrary {@link IndexDeletionPolicy}, this gives you the freedom to continue using whatever {@link IndexDeletionPolicy} you would normally want to use with yourindex. Note that you can re-use a single instance of SnapshotDeletionPolicy across multiple writers as long as they are against the same index Directory. Any snapshot held when a writer is closed will "survive" when the next writer is opened.
WARNING: This API is a new and experimental and may suddenly change.

        try
        {
            Directory dir = getDirectory( baseStorePath, identifier );
            directoryExists( dir );
            IndexType type = getType( identifier );
            SnapshotDeletionPolicy deletionPolicy = new MultipleBackupDeletionPolicy();
            IndexWriter indexWriter = new IndexWriter( dir, type.analyzer, deletionPolicy, MaxFieldLength.UNLIMITED );
            writer = Triplet.of( indexWriter, new AtomicBoolean(), deletionPolicy );
            Similarity similarity = type.getSimilarity();
            if ( similarity != null )
            {

View Full Code Here

    {
        final Collection<File> files = new ArrayList<File>();
        final Collection<SnapshotDeletionPolicy> snapshots = new ArrayList<SnapshotDeletionPolicy>();
        for ( Map.Entry<IndexIdentifier, Triplet<IndexWriter, AtomicBoolean, SnapshotDeletionPolicy>> writer : indexWriters.entrySet() )
        {
            SnapshotDeletionPolicy deletionPolicy = writer.getValue().third();
            File indexDirectory = getFileDirectory( baseStorePath, writer.getKey() );
            for ( String fileName : deletionPolicy.snapshot().getFileNames() )
            {
                files.add( new File( indexDirectory, fileName ) );
            }
            snapshots.add( deletionPolicy );
        }
        files.add( providerStore.getFile() );
        return new ClosableIterable<File>()
        {
            public Iterator<File> iterator()
            {
                return files.iterator();
            }


            public void close()
            {
                for ( SnapshotDeletionPolicy deletionPolicy : snapshots )
                {
                    deletionPolicy.release();
                }
            }
        };
    }

View Full Code Here

  }


  public void testReuseAcrossWriters() throws Exception {
    Directory dir = new MockRAMDirectory();


    SnapshotDeletionPolicy dp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
    IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), dp, IndexWriter.MaxFieldLength.UNLIMITED);
    // Force frequent flushes
    writer.setMaxBufferedDocs(2);
    Document doc = new Document();
    doc.add(new Field("content", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
    for(int i=0;i<7;i++) {
      writer.addDocument(doc);
      if (i % 2 == 0) {
        writer.commit();
      }
    }
    IndexCommit cp = (IndexCommit) dp.snapshot();
    copyFiles(dir, cp);
    writer.close();
    copyFiles(dir, cp);
    
    writer = new IndexWriter(dir, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), dp, IndexWriter.MaxFieldLength.UNLIMITED);
    copyFiles(dir, cp);
    for(int i=0;i<7;i++) {
      writer.addDocument(doc);
      if (i % 2 == 0) {
        writer.commit();
      }
    }
    copyFiles(dir, cp);
    writer.close();
    copyFiles(dir, cp);
    dp.release();
    writer = new IndexWriter(dir, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), dp, IndexWriter.MaxFieldLength.UNLIMITED);
    writer.close();
    try {
      copyFiles(dir, cp);
      fail("did not hit expected IOException");

View Full Code Here


  private void runTest(Directory dir) throws Exception {
    // Run for ~7 seconds
    final long stopTime = System.currentTimeMillis() + 7000;


    SnapshotDeletionPolicy dp = new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
    final IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(org.apache.lucene.util.Version.LUCENE_CURRENT), dp, IndexWriter.MaxFieldLength.UNLIMITED);


    // Force frequent flushes
    writer.setMaxBufferedDocs(2);

View Full Code Here

  protected SnapshotDeletionPolicy getDeletionPolicy() throws IOException {
    return getDeletionPolicy(null);
  }


  protected SnapshotDeletionPolicy getDeletionPolicy(Map<String, String> snapshots) throws IOException {
    return new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), snapshots);
  }

View Full Code Here


  private void runTest(Random random, Directory dir) throws Exception {
    // Run for ~1 seconds
    final long stopTime = System.currentTimeMillis() + 1000;


    SnapshotDeletionPolicy dp = getDeletionPolicy();
    final IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
        TEST_VERSION_CURRENT, 
        new StandardAnalyzer(TEST_VERSION_CURRENT)).setIndexDeletionPolicy(dp)
        .setMaxBufferedDocs(2));
    writer.commit();

View Full Code Here


  
  @Test
  public void testBasicSnapshots() throws Exception {
    int numSnapshots = 3;
    SnapshotDeletionPolicy sdp = getDeletionPolicy();
    
    // Create 3 snapshots: snapshot0, snapshot1, snapshot2
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
    prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot");
    writer.close();
    
    assertSnapshotExists(dir, sdp, numSnapshots);


    // open a reader on a snapshot - should succeed.
    IndexReader.open(sdp.getSnapshot("snapshot0"), true).close();


    // open a new IndexWriter w/ no snapshots to keep and assert that all snapshots are gone.
    sdp = getDeletionPolicy();
    writer = new IndexWriter(dir, getConfig(random, sdp));
    writer.deleteUnusedFiles();
    writer.close();
    assertEquals("no snapshots should exist", 1, IndexReader.listCommits(dir).size());
    
    for (int i = 0; i < numSnapshots; i++) {
      try {
        sdp.getSnapshot("snapshot" + i);
        fail("snapshot shouldn't have existed, but did: snapshot" + i);
      } catch (IllegalStateException e) {
        // expected - snapshot should not exist
      }
    }

View Full Code Here

  }


  @Test
  public void testMultiThreadedSnapshotting() throws Exception {
    Directory dir = newDirectory();
    final SnapshotDeletionPolicy sdp = getDeletionPolicy();
    final IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));


    Thread[] threads = new Thread[10];
    for (int i = 0; i < threads.length; i++) {
      threads[i] = new Thread() {
        @Override
        public void run() {
          try {
            writer.addDocument(new Document());
            writer.commit();
            sdp.snapshot(getName());
          } catch (Exception e) {
            throw new RuntimeException(e);
          }
        }
      };
      threads[i].setName("t" + i);
    }
    
    for (Thread t : threads) {
      t.start();
    }
    
    for (Thread t : threads) {
      t.join();
    }


    // Do one last commit, so that after we release all snapshots, we stay w/ one commit
    writer.addDocument(new Document());
    writer.commit();
    
    for (Thread t : threads) {
      sdp.release(t.getName());
      writer.deleteUnusedFiles();
    }
    assertEquals(1, IndexReader.listCommits(dir).size());
    writer.close();
    dir.close();

View Full Code Here


  @Test
  public void testRollbackToOldSnapshot() throws Exception {
    int numSnapshots = 2;
    Directory dir = newDirectory();
    SnapshotDeletionPolicy sdp = getDeletionPolicy();
    IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
    prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot");
    writer.close();


    // now open the writer on "snapshot0" - make sure it succeeds
    writer = new IndexWriter(dir, getConfig(random, sdp).setIndexCommit(sdp.getSnapshot("snapshot0")));
    // this does the actual rollback
    writer.commit();
    writer.deleteUnusedFiles();
    assertSnapshotExists(dir, sdp, numSnapshots - 1);
    writer.close();
    
    // but 'snapshot1' files will still exist (need to release snapshot before they can be deleted).
    String segFileName = sdp.getSnapshot("snapshot1").getSegmentsFileName();
    assertTrue("snapshot files should exist in the directory: " + segFileName, dir.fileExists(segFileName));
    dir.close();
  }

View Full Code Here

  }


  @Test
  public void testReleaseSnapshot() throws Exception {
    Directory dir = newDirectory();
    SnapshotDeletionPolicy sdp = getDeletionPolicy();
    IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
    prepareIndexAndSnapshots(sdp, writer, 1, "snapshot");
    
    // Create another commit - we must do that, because otherwise the "snapshot"
    // files will still remain in the index, since it's the last commit.
    writer.addDocument(new Document());
    writer.commit();
    
    // Release
    String snapId = "snapshot0";
    String segFileName = sdp.getSnapshot(snapId).getSegmentsFileName();
    sdp.release(snapId);
    try {
      sdp.getSnapshot(snapId);
      fail("should not have succeeded to get an unsnapshotted id");
    } catch (IllegalStateException e) {
      // expected
    }
    assertNull(sdp.getSnapshots().get(snapId));
    writer.deleteUnusedFiles();
    writer.close();
    assertFalse("segments file should not be found in dirctory: " + segFileName, dir.fileExists(segFileName));
    dir.close();
  }

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.apache.lucene.index.SnapshotDeletionPolicy

org.apache.blur.manager.writer.BlurNRTIndex

org.apache.lucene.index.TestSnapshotDeletionPolicy

org.apache.lucene.replicator.http.HttpReplicatorTest

org.apache.lucene.replicator.IndexAndTaxonomyReplicationClientTest

org.apache.lucene.replicator.IndexAndTaxonomyRevision$SnapshotDirectoryTaxonomyWriter

org.apache.lucene.replicator.IndexAndTaxonomyRevisionTest

org.apache.lucene.replicator.IndexReplicationClientTest

org.apache.lucene.replicator.IndexRevisionTest

org.apache.lucene.replicator.LocalReplicatorTest

org.apache.lucene.replicator.SessionTokenTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.