Examples of EditLogOutputStream

com.alimama.mdrill.editlog.write.EditLogOutputStream
org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream
A generic abstract class to support journaling of edits logs into a persistent storage.

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

    bkjm.format(nsi);


    long txid = 1;
    for (long i = 0; i < 3; i++) {
      long start = txid;
      EditLogOutputStream out = bkjm.startLogSegment(start);
      for (long j = 1 ; j <= DEFAULT_SEGMENT_SIZE; j++) {
        FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
        op.setTransactionId(txid++);
        out.write(op);
      }
      out.close();
      bkjm.finalizeLogSegment(start, txid-1);
      assertNotNull(
          zkc.exists(bkjm.finalizedLedgerZNode(start, txid-1), false));
    }
    zkc.delete(bkjm.finalizedLedgerZNode(DEFAULT_SEGMENT_SIZE+1,

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

    bkjm.format(nsi);


    long txid = 1;
    for (long i = 0; i < 3; i++) {
      long start = txid;
      EditLogOutputStream out = bkjm.startLogSegment(start);
      for (long j = 1 ; j <= DEFAULT_SEGMENT_SIZE; j++) {
        FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
        op.setTransactionId(txid++);
        out.write(op);
      }
      
      out.close();
      bkjm.finalizeLogSegment(start, (txid-1));
      assertNotNull(
          zkc.exists(bkjm.finalizedLedgerZNode(start, (txid-1)), false));
    }
    long start = txid;
    EditLogOutputStream out = bkjm.startLogSegment(start);
    for (long j = 1 ; j <= DEFAULT_SEGMENT_SIZE/2; j++) {
      FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
      op.setTransactionId(txid++);
      out.write(op);
    }
    out.setReadyToFlush();
    out.flush();
    out.abort();
    out.close();
    
    long numTrans = bkjm.getNumberOfTransactions(1, true);
    assertEquals((txid-1), numTrans);
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

        BKJMUtil.createJournalURI("/hdfsjournal-restartFrom1"), nsi);
    bkjm.format(nsi);


    long txid = 1;
    long start = txid;
    EditLogOutputStream out = bkjm.startLogSegment(txid);
    for (long j = 1 ; j <= DEFAULT_SEGMENT_SIZE; j++) {
      FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
      op.setTransactionId(txid++);
      out.write(op);
    }
    out.close();
    bkjm.finalizeLogSegment(start, (txid-1));
    
    txid = 1;
    try {
      out = bkjm.startLogSegment(txid);
      fail("Shouldn't be able to start another journal from " + txid
          + " when one already exists");
    } catch (Exception ioe) {
      LOG.info("Caught exception as expected", ioe);
    }


    // test border case
    txid = DEFAULT_SEGMENT_SIZE;
    try {
      out = bkjm.startLogSegment(txid);
      fail("Shouldn't be able to start another journal from " + txid
          + " when one already exists");
    } catch (IOException ioe) {
      LOG.info("Caught exception as expected", ioe);
    }


    // open journal continuing from before
    txid = DEFAULT_SEGMENT_SIZE + 1;
    start = txid;
    out = bkjm.startLogSegment(start);
    assertNotNull(out);


    for (long j = 1 ; j <= DEFAULT_SEGMENT_SIZE; j++) {
      FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
      op.setTransactionId(txid++);
      out.write(op);
    }
    out.close();
    bkjm.finalizeLogSegment(start, (txid-1));


    // open journal arbitarily far in the future
    txid = DEFAULT_SEGMENT_SIZE * 4;
    out = bkjm.startLogSegment(txid);

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream


    BookKeeperJournalManager bkjm2 = new BookKeeperJournalManager(conf,
        BKJMUtil.createJournalURI("/hdfsjournal-dualWriter"), nsi);




    EditLogOutputStream out1 = bkjm1.startLogSegment(start);
    try {
      bkjm2.startLogSegment(start);
      fail("Shouldn't have been able to open the second writer");
    } catch (IOException ioe) {
      LOG.info("Caught exception as expected", ioe);
    }finally{
      out1.close();
    }
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

        BKJMUtil.createJournalURI("/hdfsjournal-simpleread"),
        nsi);
    bkjm.format(nsi);


    final long numTransactions = 10000;
    EditLogOutputStream out = bkjm.startLogSegment(1);
    for (long i = 1 ; i <= numTransactions; i++) {
      FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
      op.setTransactionId(i);
      out.write(op);
    }
    out.close();
    bkjm.finalizeLogSegment(1, numTransactions);


    List<EditLogInputStream> in = new ArrayList<EditLogInputStream>();
    bkjm.selectInputStreams(in, 1, true);
    try {

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

    BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
        BKJMUtil.createJournalURI("/hdfsjournal-simplerecovery"),
        nsi);
    bkjm.format(nsi);


    EditLogOutputStream out = bkjm.startLogSegment(1);
    for (long i = 1 ; i <= 100; i++) {
      FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
      op.setTransactionId(i);
      out.write(op);
    }
    out.setReadyToFlush();
    out.flush();


    out.abort();
    out.close();




    assertNull(zkc.exists(bkjm.finalizedLedgerZNode(1, 100), false));
    assertNotNull(zkc.exists(bkjm.inprogressZNode(1), false));

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

      NamespaceInfo nsi = newNSInfo();
      BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
          BKJMUtil.createJournalURI("/hdfsjournal-allbookiefailure"),
          nsi);
      bkjm.format(nsi);
      EditLogOutputStream out = bkjm.startLogSegment(txid);


      for (long i = 1 ; i <= 3; i++) {
        FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
        op.setTransactionId(txid++);
        out.write(op);
      }
      out.setReadyToFlush();
      out.flush();
      bookieToFail.shutdown();
      assertEquals("New bookie didn't die",
                   numBookies, bkutil.checkBookiesUp(numBookies, 10));


      try {
        for (long i = 1 ; i <= 3; i++) {
          FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
          op.setTransactionId(txid++);
          out.write(op);
        }
        out.setReadyToFlush();
        out.flush();
        fail("should not get to this stage");
      } catch (IOException ioe) {
        LOG.debug("Error writing to bookkeeper", ioe);
        assertTrue("Invalid exception message",
                   ioe.getMessage().contains("Failed to write to bookkeeper"));
      }
      replacementBookie = bkutil.newBookie();


      assertEquals("New bookie didn't start",
                   numBookies+1, bkutil.checkBookiesUp(numBookies+1, 10));
      bkjm.recoverUnfinalizedSegments();
      out = bkjm.startLogSegment(txid);
      for (long i = 1 ; i <= 3; i++) {
        FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
        op.setTransactionId(txid++);
        out.write(op);
      }


      out.setReadyToFlush();
      out.flush();


    } catch (Exception e) {
      LOG.error("Exception in test", e);
      throw e;
    } finally {

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

      BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
          BKJMUtil.createJournalURI("/hdfsjournal-onebookiefailure"),
          nsi);
      bkjm.format(nsi);


      EditLogOutputStream out = bkjm.startLogSegment(txid);
      for (long i = 1 ; i <= 3; i++) {
        FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
        op.setTransactionId(txid++);
        out.write(op);
      }
      out.setReadyToFlush();
      out.flush();


      replacementBookie = bkutil.newBookie();
      assertEquals("replacement bookie didn't start",
                   ensembleSize+1, bkutil.checkBookiesUp(ensembleSize+1, 10));
      bookieToFail.shutdown();
      assertEquals("New bookie didn't die",
                   ensembleSize, bkutil.checkBookiesUp(ensembleSize, 10));


      for (long i = 1 ; i <= 3; i++) {
        FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
        op.setTransactionId(txid++);
        out.write(op);
      }
      out.setReadyToFlush();
      out.flush();
    } catch (Exception e) {
      LOG.error("Exception in test", e);
      throw e;
    } finally {
      if (replacementBookie != null) {

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

    NamespaceInfo nsi = newNSInfo();
    BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri,
                                                                 nsi);
    bkjm.format(nsi);


    EditLogOutputStream out = bkjm.startLogSegment(1);
    for (long i = 1; i <= 100; i++) {
      FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
      op.setTransactionId(i);
      out.write(op);
    }
    out.close();
    bkjm.finalizeLogSegment(1, 100);


    out = bkjm.startLogSegment(101);
    out.close();
    bkjm.close();
    String inprogressZNode = bkjm.inprogressZNode(101);
    zkc.setData(inprogressZNode, new byte[0], -1);


    bkjm = new BookKeeperJournalManager(conf, uri, nsi);

View Full Code Here

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream

    NamespaceInfo nsi = newNSInfo();
    BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf, uri,
                                                                 nsi);
    bkjm.format(nsi);


    EditLogOutputStream out = bkjm.startLogSegment(1);
    for (long i = 1; i <= 100; i++) {
      FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
      op.setTransactionId(i);
      out.write(op);
    }
    out.close();
    bkjm.finalizeLogSegment(1, 100);


    out = bkjm.startLogSegment(101);
    out.close();
    bkjm.close();


    String inprogressZNode = bkjm.inprogressZNode(101);
    zkc.setData(inprogressZNode, "WholeLottaJunk".getBytes(), -1);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.