Examples of FastCopy


Examples of org.apache.hadoop.hdfs.tools.FastCopy

      skipCRCCheck = job.getBoolean(Options.SKIPCRC.propertyname, false);
      useFastCopy = job.getBoolean(Options.USEFASTCOPY.propertyname, false);

      if (useFastCopy) {
        try {
          fc = new FastCopy(job);
        } catch (Exception e) {
          LOG.error("Exception during fastcopy instantiation", e);
        }
      }
      this.job = job;
View Full Code Here

Examples of org.apache.hadoop.hdfs.tools.FastCopy

  @Test
  public void testBadDatanode() throws Exception {
    DFSTestUtil util = new DFSTestUtil("testBadDatanode", 3, 1, MAX_FILE_SIZE);
    String topDir = "/testBadDatanode";
    util.createFiles(fs, topDir);
    FastCopy fastCopy = new FastCopy(conf);
    cluster.shutdownDataNode(0, true);
    try {
      for (String fileName : util.getFileNames(topDir)) {
        fastCopy.copy(fileName, fileName + "dst", (DistributedFileSystem) fs,
            (DistributedFileSystem) fs);
      }
      Map<DatanodeInfo, Integer> dnErrors = fastCopy.getDatanodeErrors();
      assertEquals(1, dnErrors.size());
      int errors = dnErrors.values().iterator().next();
      assertTrue(errors >= conf.getInt("dfs.fastcopy.max.datanode.errors", 3) + 1);
    } finally {
      fastCopy.shutdown();
    }
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.tools.FastCopy

  @Test(timeout = 2 * 60 * 1000)
  public void testDeadDatanodes() throws Exception {
    DFSTestUtil util = new DFSTestUtil("testDeadDatanodes", 1, 1, MAX_FILE_SIZE);
    String topDir = "/testDeadDatanodes";
    util.createFiles(fs, topDir);
    FastCopy fastCopy = new FastCopy(conf);

    // Find the locations for the last block of the file.
    String filename = util.getFileNames(topDir)[0];
    LocatedBlocks lbks = cluster.getNameNode().getBlockLocations(filename, 0,
        Long.MAX_VALUE);
    assertNotNull(lbks);
    LocatedBlock lastBlock = lbks.get(lbks.locatedBlockCount() - 1);

    // Shutdown all datanodes that have the last block of the file.
    for (DatanodeInfo dnInfo : lastBlock.getLocations()) {
      InetSocketAddress addr = new InetSocketAddress(dnInfo.getHost(),
          dnInfo.getPort());
      for (int i = 0; i < cluster.getDataNodes().size(); i++) {
        DataNode dn = cluster.getDataNodes().get(i);
        if (dn.getSelfAddr().equals(addr)) {
          cluster.shutdownDataNode(i, true);
        }
      }
    }

    // Now run FastCopy
    try {
      for (String fileName : util.getFileNames(topDir)) {
        fastCopy.copy(fileName, fileName + "dst", (DistributedFileSystem) fs,
            (DistributedFileSystem) fs);
      }

    } catch (ExecutionException e) {
      System.out.println("Expected exception : " + e);
      assertEquals(IOException.class, e.getCause().getClass());
      return;
    } finally {
      fastCopy.shutdown();
    }
    fail("No exception thrown");
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.tools.FastCopy

  public void testCleanShutdown() throws Exception {
    String filename = "/testCleanShutdown";
    DFSTestUtil.createFile(fs, new Path(filename), MAX_FILE_SIZE, (short) 3,
        System.currentTimeMillis());

    FastCopy fastCopy = new FastCopy(conf);
    try {
      fastCopy.copy(filename, filename + "dst", (DistributedFileSystem) fs,
          (DistributedFileSystem) fs);
    } finally {
      fastCopy.shutdown();
    }
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.tools.FastCopy

  public void testFastCopy(boolean hardlink) throws Exception {
    // Create a source file.
    String src = "/testFastCopySrc" + hardlink;
    generateRandomFile(fs, src, FILESIZE);
    String destination = "/testFastCopyDestination" + hardlink;
    FastCopy fastCopy = new FastCopy(conf);
    NameNode namenode = cluster.getNameNode();
    try {
      for (int i = 0; i < COPIES; i++) {
        fastCopy.copy(src, destination + i, fs, fs);
        assertTrue(verifyCopiedFile(src, destination + i, namenode, namenode,
            fs, fs, hardlink));
        verifyFileStatus(destination + i, namenode, fastCopy);
      }
    } catch (Exception e) {
      LOG.error("Fast Copy failed with exception : ", e);
      fail("Fast Copy failed");
    } finally {
      fastCopy.shutdown();
    }
    assertTrue(pass);
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.tools.FastCopy

  public void testFastCopyOldAPI(boolean hardlink) throws Exception {
    // Create a source file.
    String src = "/testFastCopySrc" + hardlink;
    generateRandomFile(fs, src, FILESIZE);
    String destination = "/testFastCopyDestination" + hardlink;
    FastCopy fastCopy = new FastCopy(conf, fs, fs);
    NameNode namenode = cluster.getNameNode();
    try {
      for (int i = 0; i < COPIES; i++) {
        fastCopy.copy(src, destination + i);
        assertTrue(verifyCopiedFile(src, destination + i, namenode, namenode,
            fs, fs, hardlink));
        verifyFileStatus(destination + i, namenode, fastCopy);
      }
    } catch (Exception e) {
      LOG.error("Fast Copy failed with exception : ", e);
      fail("Fast Copy failed");
    } finally {
      fastCopy.shutdown();
    }
    assertTrue(pass);
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.tools.FastCopy

  public void testFastCopyMultiple(boolean hardlink) throws Exception {
    // Create a source file.
    String src = "/testFastCopyMultipleSrc" + hardlink;
    generateRandomFile(fs, src, FILESIZE);
    String destination = "/testFastCopyMultipleDestination" + hardlink;
    FastCopy fastCopy = new FastCopy(conf);
    List<FastFileCopyRequest> requests = new ArrayList<FastFileCopyRequest>();
    for (int i = 0; i < COPIES; i++) {
      requests.add(new FastFileCopyRequest(src, destination + i, fs, fs));
    }
    NameNode namenode = cluster.getNameNode();
    try {
      fastCopy.copy(requests);
      for (FastFileCopyRequest r : requests) {
        assertTrue(verifyCopiedFile(r.getSrc(), r.getDestination(), namenode,
            namenode, fs, fs, hardlink));
        verifyFileStatus(r.getDestination(), namenode, fastCopy);
      }
    } catch (Exception e) {
      LOG.error("Fast Copy failed with exception : ", e);
      fail("Fast Copy failed");
    } finally {
      fastCopy.shutdown();
    }
    assertTrue(pass);
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.tools.FastCopy

  public void testInterFileSystemFastCopy(boolean hardlink) throws Exception {
    // Create a source file.
    String src = "/testInterFileSystemFastCopySrc" + hardlink;
    generateRandomFile(fs, src, FILESIZE);
    String destination = "/testInterFileSystemFastCopyDst" + hardlink;
    FastCopy fastCopy = new FastCopy(conf);
    NameNode srcNameNode = cluster.getNameNode();
    NameNode dstNameNode = remoteCluster.getNameNode();
    try {
      for (int i = 0; i < COPIES; i++) {
        fastCopy.copy(src, destination + i, fs, remoteFs);
        assertTrue(verifyCopiedFile(src, destination + i, srcNameNode,
            dstNameNode, fs, remoteFs, hardlink));
        verifyFileStatus(destination + i, dstNameNode, fastCopy);
      }
    } catch (Exception e) {
      LOG.error("Fast Copy failed with exception : ", e);
      fail("Fast Copy failed");
    } finally {
      fastCopy.shutdown();
    }
    assertTrue(pass);
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.tools.FastCopy

    // Create a source file.
    String src = "/testInterFileSystemFastCopyMultipleSrc" + hardlink;
    generateRandomFile(fs, src, FILESIZE);
    String destination = "/testInterFileSystemFastCopyMultipleDestination"
        + hardlink;
    FastCopy fastCopy = new FastCopy(conf);
    List<FastFileCopyRequest> requests = new ArrayList<FastFileCopyRequest>();
    for (int i = 0; i < COPIES; i++) {
      requests.add(new FastFileCopyRequest(src, destination + i, fs, remoteFs));
    }
    NameNode srcNameNode = cluster.getNameNode();
    NameNode dstNameNode = remoteCluster.getNameNode();
    try {
      fastCopy.copy(requests);
      for (FastFileCopyRequest r : requests) {
        assertTrue(verifyCopiedFile(r.getSrc(), r.getDestination(),
            srcNameNode, dstNameNode, fs, remoteFs, hardlink));
        verifyFileStatus(r.getDestination(), dstNameNode, fastCopy);
      }
    } catch (Exception e) {
      LOG.error("Fast Copy failed with exception : ", e);
      fail("Fast Copy failed");
    } finally {
      fastCopy.shutdown();
    }
    assertTrue(pass);
  }
View Full Code Here

Examples of org.apache.hadoop.hdfs.tools.FastCopy

      skipUnderConstructionFile =
          job.getBoolean(Options.SKIPUNDERCONSTRUCTION.propertyname, false);
     
      if (useFastCopy) {
        try {
          fc = new FastCopy(job, skipUnderConstructionFile);
        } catch (Exception e) {
          LOG.error("Exception during fastcopy instantiation", e);
        }
      }
      this.job = job;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.