Package org.apache.hadoop.hdfs.server.common

Examples of org.apache.hadoop.hdfs.server.common.StorageInfo

513
514
515
516
517
518
519
520
521
522
523
    DatanodeID dnId = DFSTestUtil.getLocalDatanodeID();
    BlockKey[] keys = new BlockKey[] { getBlockKey(2), getBlockKey(3) };
    ExportedBlockKeys expKeys = new ExportedBlockKeys(true, 9, 10,
        getBlockKey(1), keys);
    DatanodeRegistration reg = new DatanodeRegistration(dnId,
        new StorageInfo(), expKeys, "3.0.0");
    DatanodeRegistrationProto proto = PBHelper.convert(reg);
    DatanodeRegistration reg2 = PBHelper.convert(proto);
    compare(reg.getStorageInfo(), reg2.getStorageInfo());
    compare(reg.getExportedKeys(), reg2.getExportedKeys());
    compare(reg, reg2);
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
    dnManager = namenode.getNamesystem().getBlockManager().getDatanodeManager();

    // Register DNs
    for (int i=0; i < 6; i++) {
      DatanodeRegistration dnr = new DatanodeRegistration(dataNodes[i],
          new StorageInfo(), new ExportedBlockKeys(), VersionInfo.getVersion());
      dnrList.add(dnr);
      dnManager.registerDatanode(dnr);
      dataNodes[i].getStorageInfos()[0].setUtilizationForTesting(
          2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*blockSize, 0L,
          2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*blockSize, 0L);
View Full Code Here
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
      NNStorage dstImage = Mockito.mock(NNStorage.class);
      Mockito.doReturn(Lists.newArrayList(new File("/wont-be-written")))
        .when(dstImage).getFiles(
            Mockito.<NameNodeDirType>anyObject(), Mockito.anyString());
     
      Mockito.doReturn(new StorageInfo(1, 1, "X", 1).toColonSeparatedString())
        .when(dstImage).toColonSeparatedString();

      try {
        TransferFsImage.downloadImageToStorage(fsName, 0, dstImage, false);
        fail("Storage info was not verified");
View Full Code Here
176
177
178
179
180
181
182
183
184
185
186
      DatanodeID dnId = new DatanodeID(DN_IP_ADDR, DN_HOSTNAME,
          "fake-datanode-id", DN_XFER_PORT, DN_INFO_PORT, DN_INFO_SECURE_PORT,
          DN_IPC_PORT);
      long nnCTime = cluster.getNamesystem().getFSImage().getStorage()
          .getCTime();
      StorageInfo mockStorageInfo = mock(StorageInfo.class);
      doReturn(nnCTime).when(mockStorageInfo).getCTime();
      doReturn(HdfsConstants.LAYOUT_VERSION).when(mockStorageInfo)
          .getLayoutVersion();
      DatanodeRegistration dnReg = new DatanodeRegistration(dnId,
          mockStorageInfo, null, VersionInfo.getVersion());
View Full Code Here
219
220
221
222
223
224
225
226
227
228
229
          .build();
     
      NamenodeProtocols rpcServer = cluster.getNameNodeRpc();
     
      long nnCTime = cluster.getNamesystem().getFSImage().getStorage().getCTime();
      StorageInfo mockStorageInfo = mock(StorageInfo.class);
      doReturn(nnCTime).when(mockStorageInfo).getCTime();
     
      DatanodeRegistration mockDnReg = mock(DatanodeRegistration.class);
      doReturn(HdfsConstants.LAYOUT_VERSION).when(mockDnReg).getVersion();
      doReturn(123).when(mockDnReg).getXferPort();
View Full Code Here
267
268
269
270
271
272
273
274
275
276
277
          .build();
     
      NamenodeProtocols rpcServer = cluster.getNameNodeRpc();
     
      long nnCTime = cluster.getNamesystem().getFSImage().getStorage().getCTime();
      StorageInfo mockStorageInfo = mock(StorageInfo.class);
      doReturn(nnCTime).when(mockStorageInfo).getCTime();
     
      DatanodeRegistration mockDnReg = mock(DatanodeRegistration.class);
      doReturn(HdfsConstants.LAYOUT_VERSION).when(mockDnReg).getVersion();
      doReturn("fake-storage-id").when(mockDnReg).getDatanodeUuid();
View Full Code Here
227
228
229
230
231
232
233
234
        .setCTime(info.getCTime()).setLayoutVersion(info.getLayoutVersion())
        .setNamespceID(info.getNamespaceID()).build();
  }

  public static StorageInfo convert(StorageInfoProto info) {
    return new StorageInfo(info.getLayoutVersion(), info.getNamespceID(),
        info.getClusterID(), info.getCTime());
  }
View Full Code Here
705
706
707
708
709
710
711
712
713
714
715
  DatanodeRegistration createBPRegistration(NamespaceInfo nsInfo) {
    DatanodeRegistration bpRegistration = createUnknownBPRegistration();
    String blockPoolId = nsInfo.getBlockPoolID();
   
    bpRegistration.setStorageID(getStorageId());
    StorageInfo storageInfo = storage.getBPStorage(blockPoolId);
    if (storageInfo == null) {
      // it's null in the case of SimulatedDataSet
      bpRegistration.storageInfo.layoutVersion = HdfsConstants.LAYOUT_VERSION;
      bpRegistration.setStorageInfo(nsInfo);
    } else {
View Full Code Here
841
842
843
844
845
846
847
848
849
850
851
852
853
   
    if (!simulatedFSDataset) {
      // read storage info, lock data dirs and transition fs state if necessary         
      storage.recoverTransitionRead(DataNode.this, blockPoolId, nsInfo,
          dataDirs, startOpt);
      StorageInfo bpStorage = storage.getBPStorage(blockPoolId);
      LOG.info("setting up storage: nsid=" +
          bpStorage.getNamespaceID() + ";bpid="
          + blockPoolId + ";lv=" + storage.getLayoutVersion() +
          ";nsInfo=" + nsInfo);
    }
   
    // In the case that this is the first block pool to connect, initialize
View Full Code Here
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
      log("DataNode upgrade with future stored layout version in current", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE);
      baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
      UpgradeUtilities.createVersionFile(DATA_NODE, baseDirs,
                                         new StorageInfo(Integer.MIN_VALUE,
                                                         UpgradeUtilities.getCurrentNamespaceID(cluster),
                                                         UpgradeUtilities.getCurrentFsscTime(cluster)));
      startDataNodeShouldFail(StartupOption.REGULAR);
      cluster.shutdown();
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
      UpgradeUtilities.createEmptyDirs(dataNodeDirs);
     
      log("DataNode upgrade with newer fsscTime in current", numDirs);
      UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      cluster = new MiniDFSCluster(conf, 0, StartupOption.UPGRADE);
      baseDirs = UpgradeUtilities.createStorageDirs(DATA_NODE, dataNodeDirs, "current");
      UpgradeUtilities.createVersionFile(DATA_NODE, baseDirs,
                                         new StorageInfo(UpgradeUtilities.getCurrentLayoutVersion(),
                                                         UpgradeUtilities.getCurrentNamespaceID(cluster),
                                                         Long.MAX_VALUE));
      startDataNodeShouldFail(StartupOption.REGULAR);
      cluster.shutdown();
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
      UpgradeUtilities.createEmptyDirs(dataNodeDirs);

      log("NameNode upgrade with no edits file", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      for (File f : baseDirs) {
        FileUtil.fullyDelete(new File(f,"edits"));
      }
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
     
      log("NameNode upgrade with no image file", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      for (File f : baseDirs) {
        FileUtil.fullyDelete(new File(f,"fsimage"));
      }
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
     
      log("NameNode upgrade with corrupt version file", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      for (File f : baseDirs) {
        UpgradeUtilities.corruptFile(new File(f,"VERSION"));
      }
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
     
      log("NameNode upgrade with old layout version in current", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      UpgradeUtilities.createVersionFile(NAME_NODE, baseDirs,
                                         new StorageInfo(Storage.LAST_UPGRADABLE_LAYOUT_VERSION + 1,
                                                         UpgradeUtilities.getCurrentNamespaceID(null),
                                                         UpgradeUtilities.getCurrentFsscTime(null)));
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
     
      log("NameNode upgrade with future layout version in current", numDirs);
      baseDirs = UpgradeUtilities.createStorageDirs(NAME_NODE, nameNodeDirs, "current");
      UpgradeUtilities.createVersionFile(NAME_NODE, baseDirs,
                                         new StorageInfo(Integer.MIN_VALUE,
                                                         UpgradeUtilities.getCurrentNamespaceID(null),
                                                         UpgradeUtilities.getCurrentFsscTime(null)));
      startNameNodeShouldFail(StartupOption.UPGRADE);
      UpgradeUtilities.createEmptyDirs(nameNodeDirs);
    } // end numDir loop
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hdfs.server.common.StorageInfo

  • org.apache.hadoop.hdfs.DFSTestUtil
  • org.apache.hadoop.hdfs.notifier.NotifierUtils
  • org.apache.hadoop.hdfs.notifier.server.ServerLogReaderAvatar
  • org.apache.hadoop.hdfs.notifier.server.ServerLogReaderTransactional
  • org.apache.hadoop.hdfs.notifier.server.TestServerLogReaderVersion
  • org.apache.hadoop.hdfs.notifier.server.TestTransactionalEditLogReader
  • org.apache.hadoop.hdfs.protocolPB.PBHelper
  • org.apache.hadoop.hdfs.protocolPB.TestPBHelper
  • org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager
  • org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos$GetStorageStateProto
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.