Examples of EditLogOutputStream

com.alimama.mdrill.editlog.write.EditLogOutputStream
org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream
A generic abstract class to support journaling of edits logs into a persistent storage.

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

  @Test
  public void testSimpleRead() throws Exception {
    BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
        BKJMUtil.createJournalURI("/hdfsjournal-simpleread"));
    final long numTransactions = 10000;
    EditLogOutputStream out = bkjm.startLogSegment(1);
    for (long i = 1 ; i <= numTransactions; i++) {
      FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
      op.setTransactionId(i);
      out.write(op);
    }
    out.close();
    bkjm.finalizeLogSegment(1, numTransactions);


     
    EditLogInputStream in = bkjm.getInputStream(1, true);
    try {

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream


  @Test
  public void testSimpleRecovery() throws Exception {
    BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
        BKJMUtil.createJournalURI("/hdfsjournal-simplerecovery"));
    EditLogOutputStream out = bkjm.startLogSegment(1);
    for (long i = 1 ; i <= 100; i++) {
      FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
      op.setTransactionId(i);
      out.write(op);
    }
    out.setReadyToFlush();
    out.flush();


    out.abort();
    out.close();




    assertNull(zkc.exists(bkjm.finalizedLedgerZNode(1, 100), false));
    assertNotNull(zkc.exists(bkjm.inprogressZNode(1), false));

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

      conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_QUORUM_SIZE,
                  ensembleSize);
      long txid = 1;
      BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
          BKJMUtil.createJournalURI("/hdfsjournal-allbookiefailure"));
      EditLogOutputStream out = bkjm.startLogSegment(txid);


      for (long i = 1 ; i <= 3; i++) {
        FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
        op.setTransactionId(txid++);
        out.write(op);
      }
      out.setReadyToFlush();
      out.flush();
      bookieToFail.shutdown();
      assertEquals("New bookie didn't die",
                   numBookies, bkutil.checkBookiesUp(numBookies, 10));


      try {
        for (long i = 1 ; i <= 3; i++) {
          FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
          op.setTransactionId(txid++);
          out.write(op);
        }
        out.setReadyToFlush();
        out.flush();
        fail("should not get to this stage");
      } catch (IOException ioe) {
        LOG.debug("Error writing to bookkeeper", ioe);
        assertTrue("Invalid exception message",
                   ioe.getMessage().contains("Failed to write to bookkeeper"));
      }
      replacementBookie = bkutil.newBookie();


      assertEquals("New bookie didn't start",
                   numBookies+1, bkutil.checkBookiesUp(numBookies+1, 10));
      bkjm.recoverUnfinalizedSegments();
      out = bkjm.startLogSegment(txid);
      for (long i = 1 ; i <= 3; i++) {
        FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
        op.setTransactionId(txid++);
        out.write(op);
      }


      out.setReadyToFlush();
      out.flush();


    } catch (Exception e) {
      LOG.error("Exception in test", e);
      throw e;
    } finally {

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

      conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_QUORUM_SIZE,
                  ensembleSize);
      long txid = 1;
      BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
          BKJMUtil.createJournalURI("/hdfsjournal-onebookiefailure"));
      EditLogOutputStream out = bkjm.startLogSegment(txid);
      for (long i = 1 ; i <= 3; i++) {
        FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
        op.setTransactionId(txid++);
        out.write(op);
      }
      out.setReadyToFlush();
      out.flush();


      replacementBookie = bkutil.newBookie();
      assertEquals("replacement bookie didn't start",
                   ensembleSize+1, bkutil.checkBookiesUp(ensembleSize+1, 10));
      bookieToFail.shutdown();
      assertEquals("New bookie didn't die",
                   ensembleSize, bkutil.checkBookiesUp(ensembleSize, 10));


      for (long i = 1 ; i <= 3; i++) {
        FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
        op.setTransactionId(txid++);
        out.write(op);
      }
      out.setReadyToFlush();
      out.flush();
    } catch (Exception e) {
      LOG.error("Exception in test", e);
      throw e;
    } finally {
      if (replacementBookie != null) {

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

  @Test
  public void testEmptyInprogressNode() throws Exception {
    URI uri = BKJMUtil.createJournalURI("/hdfsjournal-emptyInprogress");
    BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri);


    EditLogOutputStream out = bkjm.startLogSegment(1);
    for (long i = 1; i <= 100; i++) {
      FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
      op.setTransactionId(i);
      out.write(op);
    }
    out.close();
    bkjm.finalizeLogSegment(1, 100);


    out = bkjm.startLogSegment(101);
    out.close();
    bkjm.close();
    String inprogressZNode = bkjm.inprogressZNode(101);
    zkc.setData(inprogressZNode, new byte[0], -1);


    bkjm = new BookKeeperJournalManager(conf, uri);

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

  @Test
  public void testCorruptInprogressNode() throws Exception {
    URI uri = BKJMUtil.createJournalURI("/hdfsjournal-corruptInprogress");
    BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri);


    EditLogOutputStream out = bkjm.startLogSegment(1);
    for (long i = 1; i <= 100; i++) {
      FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
      op.setTransactionId(i);
      out.write(op);
    }
    out.close();
    bkjm.finalizeLogSegment(1, 100);


    out = bkjm.startLogSegment(101);
    out.close();
    bkjm.close();


    String inprogressZNode = bkjm.inprogressZNode(101);
    zkc.setData(inprogressZNode, "WholeLottaJunk".getBytes(), -1);

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

  @Test
  public void testEmptyInprogressLedger() throws Exception {
    URI uri = BKJMUtil.createJournalURI("/hdfsjournal-emptyInprogressLedger");
    BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri);


    EditLogOutputStream out = bkjm.startLogSegment(1);
    for (long i = 1; i <= 100; i++) {
      FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
      op.setTransactionId(i);
      out.write(op);
    }
    out.close();
    bkjm.finalizeLogSegment(1, 100);


    out = bkjm.startLogSegment(101);
    out.close();
    bkjm.close();


    bkjm = new BookKeeperJournalManager(conf, uri);
    bkjm.recoverUnfinalizedSegments();
    out = bkjm.startLogSegment(101);
    for (long i = 1; i <= 100; i++) {
      FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
      op.setTransactionId(i);
      out.write(op);
    }
    out.close();
    bkjm.finalizeLogSegment(101, 200);


    bkjm.close();
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

  public void testRefinalizeAlreadyFinalizedInprogress() throws Exception {
    URI uri = BKJMUtil
        .createJournalURI("/hdfsjournal-refinalizeInprogressLedger");
    BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri);


    EditLogOutputStream out = bkjm.startLogSegment(1);
    for (long i = 1; i <= 100; i++) {
      FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
      op.setTransactionId(i);
      out.write(op);
    }
    out.close();
    bkjm.close();


    String inprogressZNode = bkjm.inprogressZNode(1);
    String finalizedZNode = bkjm.finalizedLedgerZNode(1, 100);
    assertNotNull("inprogress znode doesn't exist", zkc.exists(inprogressZNode,

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

  }


  private String startAndFinalizeLogSegment(BookKeeperJournalManager bkjm,
      int startTxid, int endTxid) throws IOException, KeeperException,
      InterruptedException {
    EditLogOutputStream out = bkjm.startLogSegment(startTxid);
    for (long i = startTxid; i <= endTxid; i++) {
      FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
      op.setTransactionId(i);
      out.write(op);
    }
    out.close();
    // finalize the inprogress_1 log segment.
    bkjm.finalizeLogSegment(startTxid, endTxid);
    String zkpath1 = bkjm.finalizedLedgerZNode(startTxid, endTxid);
    assertNotNull(zkc.exists(zkpath1, false));
    assertNull(zkc.exists(bkjm.inprogressZNode(startTxid), false));

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

  @Test
  public void testCrashAtBeginningOfSegment() throws Exception {
    writeSegment(cluster, qjm, 1, 3, true);
    waitForAllPendingCalls(qjm.getLoggerSetForTests());
    
    EditLogOutputStream stm = qjm.startLogSegment(4);
    try {
      waitForAllPendingCalls(qjm.getLoggerSetForTests());
    } finally {
      stm.abort();
    }
    
    
    // Make a new QJM
    qjm = closeLater(new QuorumJournalManager(

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.