Package org.apache.hadoop.hdfs.server.namenode

Examples of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

153
154
155
156
157
158
159
160
161
162
163
            fsimage.dirIterator(NameNodeDirType.EDITS); it.hasNext();) {
      File editFile = FSImage.getImageFile(it.next(), NameNodeFile.EDITS);
      // Start from 0 when loading edit logs.
      editLog.setStartTransactionId(0);
      System.out.println("Verifying file: " + editFile);
      EditLogInputStream is = new EditLogFileInputStream(editFile);
      FSEditLogLoader loader = new FSEditLogLoader(namesystem);
      int numEdits = loader.loadFSEdits(is, namesystem.getEditLog().getCurrentTxId());
      int numLeases = namesystem.leaseManager.countLease();
      System.out.println("Number of outstanding leases " + numLeases);
      assertEquals(0, numLeases);
View Full Code Here
209
210
211
212
213
214
215
216
217
218
219
          continue;
       
        cluster.getNameNode().getFSImage().getEditLog().setStartTransactionId(0);
       
        System.out.println("Verifying file: " + editFile);
        EditLogInputStream is = new EditLogFileInputStream(editFile);
        FSEditLogLoader loader = new FSEditLogLoader(namesystem);
        int numEdits = loader.loadFSEdits(is, namesystem.getEditLog().getCurrentTxId());
        int numLeases = namesystem.leaseManager.countLease();
        System.out.println("Number of outstanding leases " + numLeases);
        assertEquals(0, numLeases);
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
        File file = null;
        EditLogInputStream elis = null;
        OfflineEditsLoader loader = null;
        try {
          file = new File(inputFileName);
          elis = new EditLogFileInputStream(file, HdfsConstants.INVALID_TXID,
              HdfsConstants.INVALID_TXID, false);
          loader = new OfflineEditsBinaryLoader(visitor, elis, flags);
        } finally {
          if ((loader == null) && (elis != null)) {
            elis.close();
View Full Code Here
327
328
329
330
331
332
333
334
335
336
337
            3 + expectedTxns - 1);
        assertTrue("Expect " + editFile + " exists", editFile.exists());
       
        System.out.println("Verifying file: " + editFile);
        int numEdits = loader.loadFSEdits(
            new EditLogFileInputStream(editFile), 3);
        int numLeases = namesystem.leaseManager.countLease();
        System.out.println("Number of outstanding leases " + numLeases);
        assertEquals(0, numLeases);
        assertTrue("Verification for " + editFile + " failed. " +
                   "Expected " + expectedTxns + " transactions. "+
View Full Code Here
314
315
316
317
318
319
320
321
322
323
324
            3 + expectedTxns - 1);
        assertTrue("Expect " + editFile + " exists", editFile.exists());
       
        System.out.println("Verifying file: " + editFile);
        int numEdits = loader.loadFSEdits(
            new EditLogFileInputStream(editFile), 3);
        int numLeases = namesystem.leaseManager.countLease();
        System.out.println("Number of outstanding leases " + numLeases);
        assertEquals(0, numLeases);
        assertTrue("Verification for " + editFile + " failed. " +
                   "Expected " + expectedTxns + " transactions. "+
View Full Code Here
350
351
352
353
354
355
356
357
358
359
360
   
    LOG.info("Trying to load the edit log from " + txFile.getAbsolutePath());
   
    do {
      try {
        inputStream = new EditLogFileInputStream(txFile);
        editLogFilePosition = inputStream.getPosition();
        curStreamConsumed = false;
        curStreamFinished = false;
        readNullAfterStreamFinished = false;
        LOG.info("Successfully loaded the edits log from " +
View Full Code Here
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
            ledgerId,
            0,
            1,
            numEdits,
            false);
    EditLogFileInputStream fileEditsIn =
        new EditLogFileInputStream(tempEditsFile);

    assertEquals("Length in bytes must be equal!",
        bkEditsIn.length(), fileEditsIn.length());

    FSEditLogTestUtil.assertStreamsAreEquivalent(numEdits,
        ImmutableMap.of("BookKeeper", bkEditsIn, "File", fileEditsIn));
    assertNull("BookKeeper edit log must end at txid 100", bkEditsIn.readOp());
  }
View Full Code Here
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
            0,
            1,
            numEdits,
            false);

    EditLogFileInputStream fileEditsIn =
        new EditLogFileInputStream(tempEditsFile);

    assertEquals("Length in bytes must be equal!",
        bkEditsIn.length(), fileEditsIn.length());

    long lastBkPos = bkEditsIn.getPosition();
    long lastFilePos = fileEditsIn.getPosition();
    for (int i = 1; i <= numEdits; i++) {
      assertEquals("Position in file must be equal position in bk",
          lastBkPos, lastFilePos);
      bkEditsIn.refresh(lastBkPos, -1);
      fileEditsIn.refresh(lastFilePos, -1);
      FSEditLogOp opFromBk = bkEditsIn.readOp();
      FSEditLogOp opFromFile = fileEditsIn.readOp();
      if (LOG.isDebugEnabled()) {
        LOG.debug("txId = " + i + ", " + "opFromBk = " + opFromBk +
            ", opFromFile = " + opFromFile);
      }
      assertEquals(
          "Operation read from file and BookKeeper must be same after refresh",
          opFromBk, opFromFile);
      lastBkPos = bkEditsIn.getPosition();
      lastFilePos = fileEditsIn.getPosition();
    }
    assertNull("BookKeeper edit log must end at last txId", bkEditsIn.readOp());
  }
View Full Code Here
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
            ledgerId,
            0,
            1,
            numEdits,
            false);
    EditLogFileInputStream fileEditsIn =
        new EditLogFileInputStream(tempEditsFile);
    long lastBkPos = bkEditsIn.getPosition();
    long lastFilePos = fileEditsIn.getPosition();
    for (int i = 0; i <= numEdits; i++) {
      assertEquals("Position in file must be equal to position in bk",
          lastBkPos, lastFilePos);
      bkEditsIn.refresh(lastBkPos, -1);
      fileEditsIn.refresh(lastFilePos, -1);
      FSEditLogOp opFromBk = bkEditsIn.readOp();
      FSEditLogOp opFromFile = fileEditsIn.readOp();
      if (LOG.isDebugEnabled()) {
        LOG.debug("txId = " + i + ", " + "opFromBk = " + opFromBk +
            ", opFromFile = " + opFromFile);
      }
      assertEquals(
          "Operation read from file and BookKeeper must be same after refresh",
          opFromBk, opFromFile);
      lastBkPos = bkEditsIn.getPosition();
      lastFilePos = fileEditsIn.getPosition();
      bkEditsIn =
          new BookKeeperEditLogInputStream(ledgerProvider,
              ledgerId,
              0,
              1,
              numEdits,
              false);
      fileEditsIn =
          new EditLogFileInputStream(tempEditsFile);
    }
    bkEditsIn.refresh(lastBkPos, -1);
    assertNull("BookKeeper edit log must end at last txId", bkEditsIn.readOp());
  }
View Full Code Here
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
          // reached the maximum number of retries
          throw e;
        }
      }
    }
    EditLogFileInputStream fileEditsIn =
        new EditLogFileInputStream(tempEditsFile);
    for (int i = 0; i <= numEdits; i++) {
      FSEditLogOp opFromBk = bkEditsIn.readOp();
      FSEditLogOp opFromFile = fileEditsIn.readOp();
      if (LOG.isDebugEnabled()) {
        LOG.debug("txId = " + i + ", " + "opFromBk = " + opFromBk +
            ", opFromFile = " + opFromFile);
      }
      assertEquals(
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream

  • org.apache.hadoop.hdfs.notifier.server.ServerLogReaderPreTransactional
  • org.apache.hadoop.hdfs.server.namenode.BackupStorage
  • org.apache.hadoop.hdfs.server.namenode.bookkeeper.TestBookKeeperEditLogInputStream
  • org.apache.hadoop.hdfs.server.namenode.bookkeeper.TestBookKeeperJournalManager
  • org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.EditLogValidation
  • org.apache.hadoop.hdfs.server.namenode.FSImage
  • org.apache.hadoop.hdfs.server.namenode.TestEditLog
  • org.apache.hadoop.hdfs.server.namenode.TestEditLog2
  • org.apache.hadoop.hdfs.server.namenode.TestEditLogRace
  • org.apache.hadoop.hdfs.server.namenode.TestSecurityTokenEditLog
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.