Examples of DataTransferThrottler

org.apache.hadoop.hdfs.util.DataTransferThrottler
a class to throttle the data transfers. This class is thread safe. It can be shared by multiple threads. The parameter bandwidthPerSec specifies the total bandwidth shared by threads.
org.apache.hadoop.util.DataTransferThrottler
a class to throttle the block transfers. This class is thread safe. It can be shared by multiple threads. The parameter bandwidthPerSec specifies the total bandwidth shared by threads.

Examples of org.apache.hadoop.hdfs.util.DataTransferThrottler

      LOG.warn("Could not open verfication log. " +
               "Verification times are not stored.");
    }
    
    synchronized (this) {
      throttler = new DataTransferThrottler(200, MAX_SCAN_RATE);
    }
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.util.DataTransferThrottler

    long transferBandwidth = conf.getLong(
        HdfsConstants.DFS_IMAGE_TRANSFER_RATE_KEY,
        HdfsConstants.DFS_IMAGE_TRANSFER_RATE_DEFAULT);


    if (transferBandwidth > 0) {
      this.imageTransferThrottler = new DataTransferThrottler(transferBandwidth);
    }
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.util.DataTransferThrottler

   */
  public final static DataTransferThrottler getThrottler(Configuration conf) {
    long transferBandwidth = 
      conf.getLong(DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_KEY,
                   DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_DEFAULT);
    DataTransferThrottler throttler = null;
    if (transferBandwidth > 0) {
      throttler = new DataTransferThrottler(transferBandwidth);
    }
    return throttler;
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.util.DataTransferThrottler

        ImageServlet.setVerificationHeadersForGet(response, editFile);
        ImageServlet.setFileNameHeaders(response, editFile);
        editFileIn = new FileInputStream(editFile);
      }
      
      DataTransferThrottler throttler = ImageServlet.getThrottler(conf);


      // send edits
      TransferFsImage.copyFileToStream(response.getOutputStream(), editFile,
          editFileIn, throttler);

View Full Code Here

Examples of org.apache.hadoop.hdfs.util.DataTransferThrottler

    FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
    long bandwidthPerSec = 1024*1024L;
    final long TOTAL_BYTES =6*bandwidthPerSec; 
    long bytesToSend = TOTAL_BYTES; 
    long start = Time.now();
    DataTransferThrottler throttler = new DataTransferThrottler(bandwidthPerSec);
    long totalBytes = 0L;
    long bytesSent = 1024*512L; // 0.5MB
    throttler.throttle(bytesSent);
    bytesToSend -= bytesSent;
    bytesSent = 1024*768L; // 0.75MB
    throttler.throttle(bytesSent);
    bytesToSend -= bytesSent;
    try {
      Thread.sleep(1000);
    } catch (InterruptedException ignored) {}
    throttler.throttle(bytesToSend);
    long end = Time.now();
    assertTrue(totalBytes*1000/(end-start)<=bandwidthPerSec);
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.util.DataTransferThrottler

   */
  private final DataTransferThrottler getThrottler(Configuration conf) {
    long transferBandwidth = 
      conf.getLong(DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_KEY,
                   DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_DEFAULT);
    DataTransferThrottler throttler = null;
    if (transferBandwidth > 0) {
      throttler = new DataTransferThrottler(transferBandwidth);
    }
    return throttler;
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.util.DataTransferThrottler

   */
  public final static DataTransferThrottler getThrottler(Configuration conf) {
    long transferBandwidth = 
      conf.getLong(DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_KEY,
                   DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_DEFAULT);
    DataTransferThrottler throttler = null;
    if (transferBandwidth > 0) {
      throttler = new DataTransferThrottler(transferBandwidth);
    }
    return throttler;
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.util.DataTransferThrottler

   */
  public final static DataTransferThrottler getThrottler(Configuration conf) {
    long transferBandwidth = 
      conf.getLong(DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_KEY,
                   DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_DEFAULT);
    DataTransferThrottler throttler = null;
    if (transferBandwidth > 0) {
      throttler = new DataTransferThrottler(transferBandwidth);
    }
    return throttler;
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.util.DataTransferThrottler

   */
  private final DataTransferThrottler getThrottler(Configuration conf) {
    long transferBandwidth = 
      conf.getLong(DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_KEY,
                   DFSConfigKeys.DFS_IMAGE_TRANSFER_RATE_DEFAULT);
    DataTransferThrottler throttler = null;
    if (transferBandwidth > 0) {
      throttler = new DataTransferThrottler(transferBandwidth);
    }
    return throttler;
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.util.DataTransferThrottler

    FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
    long bandwidthPerSec = 1024*1024L;
    final long TOTAL_BYTES =6*bandwidthPerSec; 
    long bytesToSend = TOTAL_BYTES; 
    long start = Util.now();
    DataTransferThrottler throttler = new DataTransferThrottler(bandwidthPerSec);
    long totalBytes = 0L;
    long bytesSent = 1024*512L; // 0.5MB
    throttler.throttle(bytesSent);
    bytesToSend -= bytesSent;
    bytesSent = 1024*768L; // 0.75MB
    throttler.throttle(bytesSent);
    bytesToSend -= bytesSent;
    try {
      Thread.sleep(1000);
    } catch (InterruptedException ignored) {}
    throttler.throttle(bytesToSend);
    long end = Util.now();
    assertTrue(totalBytes*1000/(end-start)<=bandwidthPerSec);
  }

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.