Package org.apache.hadoop.hdfs.server.namenode

Examples of org.apache.hadoop.hdfs.server.namenode.FSEditLogOp$DeleteOp


          BookKeeperEditLogOutputStream out =
              new BookKeeperEditLogOutputStream(ledgerOut);
          out.create(); // Writes version to the ledger output stream

          for (int i = 1; i <= numEdits; i++) {
            FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
            // Set an increasing transaction id to verify correctness
            op.setTransactionId(i);
            if (LOG.isDebugEnabled()) {
              LOG.debug("Writing " + op);
            }

            FSEditLogTestUtil.writeToStreams(op, out);

            if (i % 1000 == 0) {
              Thread.sleep(500);
              FSEditLogTestUtil.flushStreams(out);
            }
          }
          FSEditLogTestUtil.flushStreams(out);
          FSEditLogTestUtil.closeStreams(out);
        } finally {
          // Let the producer know that we've reached the end.
          finishedProducing.set(true);
        }
        return null;
      }
    };
    Callable<Void> consumerThread = new Callable<Void>() {
      @Override
      public Void call() throws Exception {
        BookKeeperEditLogInputStream in =
            new BookKeeperEditLogInputStream(ledgerProvider,
                ledgerId,
                0,
                1,
                -1,
                true);

        long numOps = 0;
        long maxTxId = -1;
        FSEditLogOp op;
        long lastPos = in.getPosition();
        do {
          op = in.readOp();
          if (op == null) { // If we've reached the end prematurely...
            Thread.sleep(1000);
            LOG.info("Refreshing to " + lastPos);

            in.refresh(lastPos, -1); // Then refresh to last known good position
          } else {
            long txId = op.getTransactionId();
            if (txId > maxTxId) {
              // Standby ingest contains similar logic: transactions
              // with ids lower than what is already read are ignored.
              numOps++;
              maxTxId = txId;
            }

            // Remember the last known safe position that we can refresh to
            lastPos = in.getPosition();
          }
        } while (op != null || !finishedProducing.get());
        Thread.sleep(1000);

        // Once producer is shutdown, scan again from last known good position
        // until the end of the ledger. This mirrors the Ingest logic (last
        // read when being quiesced).
        in.refresh(lastPos, -1);
        do {
          op = in.readOp();
          if (op != null) {
            long txId = op.getTransactionId();
            if (txId > maxTxId) {
              numOps++;
              maxTxId = txId;
            }
          }
View Full Code Here


    Callable<Void> producerThread = new Callable<Void>() {
      @Override
      public Void call() throws Exception {
        try {
          for (int i = 0; i < numEdits; i++) {
            FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
            // Set an increasing transaction id to verify correctness
            op.setTransactionId(i);
            if (LOG.isDebugEnabled()) {
              LOG.debug("Writing " + op);
            }

            FSEditLogTestUtil.writeToStreams(op, out);

            if (i % 50 == 0) {
              Thread.sleep(100);
              FSEditLogTestUtil.flushStreams(out);
            }
          }

          FSEditLogTestUtil.flushStreams(out);
          FSEditLogTestUtil.closeStreams(out);
        } finally {
          // Let the producer know that we've reached the end.
          finishedProducing.set(true);
        }
        return null;
      }
    };
    Callable<Void> consumerThread = new Callable<Void>() {
      @Override
      public Void call() throws Exception {
        List<EditLogInputStream> streams = Lists.newArrayList();
        qjm.selectInputStreams(streams, 0, true, false);
        EditLogInputStream in = streams.get(0);

        long numOps = 0;
        long maxTxId = -1;
        FSEditLogOp op;
        long lastPos = in.getPosition();
        do {
          op = in.readOp();
          if (op == null) { // If we've reached the end prematurely...
            Thread.sleep(200);
            LOG.info("Refreshing to " + lastPos);

            in.refresh(lastPos, maxTxId); // Then refresh to last known good position
          } else {
            long txId = op.getTransactionId();
            if (txId > maxTxId) {
              // Standby ingest contains similar logic: transactions
              // with ids lower than what is already read are ignored.
              numOps++;
              maxTxId = txId;
            }

            // Remember the last known safe position that we can refresh to
            lastPos = in.getPosition();
          }
        } while (op != null || !finishedProducing.get());
        Thread.sleep(1000);

        // finalize the segment, so we can read to the end
        qjm.finalizeLogSegment(0, numEdits - 1);

        // Once producer is shutdown, scan again from last known good position
        // until the end of the ledger. This mirrors the Ingest logic (last
        // read when being quiesced).
        in.refresh(lastPos, maxTxId);
        do {
          op = in.readOp();
          if (op != null) {
            long txId = op.getTransactionId();
            if (txId > maxTxId) {
              numOps++;
              maxTxId = txId;
            }
          }
View Full Code Here

    for (int expected = firstTxnId; expected <= lastTxnId; expected++) {
      if (inProgress) { // otherwise we cannot call refresh
        stream.refresh(position, expected - 1);
      }
      FSEditLogOp op = stream.readOp();
      position = stream.getPosition();

      while (op == null) {
        assertTrue("Expected to find txid " + expected + ", "
            + "but no more streams available to read from", iter.hasNext());
        stream = iter.next();
        op = stream.readOp();
      }

      assertEquals(expected, op.getTransactionId());
      assertEquals(expected, writtenTxns.get(expected).getTransactionId());
      assertEquals(op.opCode, writtenTxns.get(expected).opCode);
    }

    assertNull(stream.readOp());
View Full Code Here

    qjm.selectInputStreams(streams, 0, true, false);
   
    long lastReadTxId = -1;
    EditLogInputStream is = streams.get(0);
    for (int i = 0; i < 3; i++) {
      FSEditLogOp op = is.readOp();
      assertNotNull(op);
      lastReadTxId = op.getTransactionId();
      LOG.info("Read transaction: " + op + " with txid: "
          + op.getTransactionId());
    }
   
    // get the stream we are tailing from
    URLLogInputStream[] tailing = new URLLogInputStream[1];
    JournalNode jn = getTailingJN(is, tailing);
   
    long position = is.getPosition();
   
    // stop the node
    jn.stopAndJoin(0);
   
    // refresh the input stream
    is.refresh(position, 0);
   
    LOG.info("Checking failed stream");
    // this guy should be disabled
    // its position should be fixed
    URLLogInputStream urlis = tailing[0];
    assertTrue(urlis.isDisabled());
    assertEquals(position, urlis.getPosition());
    assertEquals(HdfsConstants.INVALID_TXID, urlis.getLastTxId());
    try {
      urlis.readOp();
      fail("This read should fail");
    } catch (IOException e) {
      LOG.info("Expected exception: ", e);
    } // expected
   
    // reads should fall back to another stream
    LOG.info("We should be able to read from the stream");
    for (int i = 0; i < 3; i++) {
      FSEditLogOp op = is.readOp();
      assertNotNull(op);
      assertEquals(++lastReadTxId, op.getTransactionId());
      LOG.info("Read transaction: " + op + " with txid: "
          + op.getTransactionId());
      position = is.getPosition();
    }
    LOG.info("Current state of the input stream: " + is.getName());
   
    // refresh again
View Full Code Here

        out = qjm.startLogSegment(i);
        currentSegmentTxid = i;
      }

      // otherwise write a random operation
      FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
      // Set an increasing transaction id to verify correctness
      op.setTransactionId(i);
      FSEditLogTestUtil.writeToStreams(op, out);

      // sync every time to ensure we send transaction by transaction
      FSEditLogTestUtil.flushStreams(out);
View Full Code Here

    if (readLastValidTxId) {
      return null; // do not advance position
    }
   
    // read the operation
    FSEditLogOp op = reader.readOp(false);
   
    // 3) no more transactions (possibly EOF reached)
    if (op == null) {
      readLastValidTxId = true;
      return null; // do not advance position
    }
   
    // 4) we read a valid transaction but beyond lastValidTxId
    if (op.getTransactionId() > lastValidTxId) {
      readLastValidTxId = true;
      LOG.info("Reached the end of valid transactions...");
      return null; // do not advance position
    }
   
    // we reached the last valid transaction
    if (op.getTransactionId() == lastValidTxId) {
      readLastValidTxId = true;
    }
   
    // 5) we have a valid transaction, advance the position
    currentPosition = tracker.getPos();
View Full Code Here

  }

  @Override
  public FSEditLogOp nextValidOp() {
    try {
      FSEditLogOp op = reader.readOp(true);
      currentPosition = tracker.getPos();
      return op;
    } catch (Throwable e) {
      LOG.error("nextValidOp: got exception while reading " + this, e);
      return null;
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hdfs.server.namenode.FSEditLogOp$DeleteOp

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.