Examples of Progressable


Examples of org.apache.hadoop.util.Progressable

  protected HRegion instantiateRegion(final HRegionInfo regionInfo)
      throws IOException {
    HRegion r = new IndexedRegion(HTableDescriptor.getTableDir(super
        .getRootDir(), regionInfo.getTableDesc().getName()), super.log, super
        .getFileSystem(), super.conf, regionInfo, super.getFlushRequester());
    r.initialize(null, new Progressable() {
      public void progress() {
        addProcessingMessage(regionInfo);
      }
    });
    return r;
View Full Code Here

Examples of org.apache.hadoop.util.Progressable

        region = new HRegion(
            HTableDescriptor.getTableDir(rootDir,
                regionInfo.getTableDesc().getName()
            ),
            this.log, this.fs, conf, regionInfo, null, this.cacheFlusher,
            new Progressable() {
              public void progress() {
                addProcessingMessage(regionInfo);
              }
            }
        );
View Full Code Here

Examples of org.apache.hadoop.util.Progressable

  protected HRegion instantiateRegion(final HRegionInfo regionInfo)
      throws IOException {
    HRegion r = new HRegion(HTableDescriptor.getTableDir(rootDir, regionInfo
        .getTableDesc().getName()), this.hlog, this.fs, conf, regionInfo,
        this.cacheFlusher);
    r.initialize(null,  new Progressable() {
      public void progress() {
        addProcessingMessage(regionInfo);
      }
    });
    return r;
View Full Code Here

Examples of org.apache.hadoop.util.Progressable

        }

        final Path file = new Path(new File("target/test/test-camel-string").getAbsolutePath());
        Configuration conf = new Configuration();
        FileSystem fs1 = FileSystem.get(file.toUri(), conf);
        ArrayFile.Writer writer = new ArrayFile.Writer(conf, fs1, "target/test/test-camel-string1", Text.class, CompressionType.NONE, new Progressable() {
            @Override
            public void progress() {
            }
        });
        Text valueWritable = new Text();
View Full Code Here

Examples of org.apache.hadoop.util.Progressable

 
  protected HRegion instantiateRegion(final HRegionInfo regionInfo)
      throws IOException {
    return new HRegion(HTableDescriptor.getTableDir(rootDir, regionInfo
        .getTableDesc().getName()), this.log, this.fs, conf, regionInfo, null,
        this.cacheFlusher, new Progressable() {
          public void progress() {
            addProcessingMessage(regionInfo);
          }
        });
  }
View Full Code Here

Examples of org.apache.hadoop.util.Progressable

  protected HRegion instantiateRegion(final HRegionInfo regionInfo)
      throws IOException {
    HRegion r = HRegion.newHRegion(HTableDescriptor.getTableDir(rootDir, regionInfo
        .getTableDesc().getName()), this.hlog, this.fs, conf, regionInfo,
        this.cacheFlusher);
    r.initialize(null,  new Progressable() {
      public void progress() {
        addProcessingMessage(regionInfo);
      }
    });
    return r;
View Full Code Here

Examples of org.apache.hadoop.util.Progressable

    String filename = "/testFileForceSync";
    boolean forceSync = true;
    DFSClient dfsClient = ((DistributedFileSystem) fileSystem).getClient();
    DFSOutputStream out = (DFSOutputStream)dfsClient.create(
        filename, FsPermission.getDefault(), true, true, REPLICATION_NUM, BLOCK_SIZE,
        new Progressable() {
          @Override
          public void progress() {
          }
        },
        BUFFER_SIZE,
View Full Code Here

Examples of org.apache.hadoop.util.Progressable

    String filename = "/testFileParallelWrite";
    boolean doParallelWrites = true;
    DFSClient dfsClient = ((DistributedFileSystem) fileSystem).getClient();
    DFSOutputStream out = (DFSOutputStream)dfsClient.create(
        filename, FsPermission.getDefault(), true, true, REPLICATION_NUM, BLOCK_SIZE,
        new Progressable() {
          @Override
          public void progress() {
          }
        },
        BUFFER_SIZE,
View Full Code Here

Examples of org.apache.hadoop.util.Progressable

   * @return true if file was reconstructed, false if no reconstruction
   * was necessary or possible.
   */
  boolean reconstructFile(Path srcPath, Context context)
      throws IOException, InterruptedException {
    Progressable progress = context;
    if (progress == null) {
      progress = RaidUtils.NULL_PROGRESSABLE;
    }

    if (RaidNode.isParityHarPartFile(srcPath)) {
View Full Code Here

Examples of org.apache.hadoop.util.Progressable

   */
  boolean processFile(Path srcPath, ParityFilePair parityPair,
      Decoder decoder, Context context) throws IOException,
      InterruptedException {
    LOG.info("Processing file " + srcPath);
    Progressable progress = context;
    if (progress == null) {
      progress = RaidUtils.NULL_PROGRESSABLE;
    }

    DistributedFileSystem srcFs = getDFS(srcPath);
    FileStatus srcStat = srcFs.getFileStatus(srcPath);
    long blockSize = srcStat.getBlockSize();
    long srcFileSize = srcStat.getLen();
    String uriPath = srcPath.toUri().getPath();

    int numBlocksReconstructed = 0;
    List<LocatedBlockWithMetaInfo> lostBlocks = lostBlocksInFile(srcFs, uriPath, srcStat);
    if (lostBlocks.size() == 0) {
      LOG.warn("Couldn't find any lost blocks in file " + srcPath +
          ", ignoring...");
      return false;
    }
    for (LocatedBlockWithMetaInfo lb: lostBlocks) {
      Block lostBlock = lb.getBlock();
      long lostBlockOffset = lb.getStartOffset();

      LOG.info("Found lost block " + lostBlock +
          ", offset " + lostBlockOffset);

      final long blockContentsSize =
        Math.min(blockSize, srcFileSize - lostBlockOffset);
      File localBlockFile =
        File.createTempFile(lostBlock.getBlockName(), ".tmp");
      localBlockFile.deleteOnExit();

      try {
        decoder.recoverBlockToFile(srcFs, srcPath, parityPair.getFileSystem(),
            parityPair.getPath(), blockSize,
            lostBlockOffset, localBlockFile,
            blockContentsSize, context);

        // Now that we have recovered the file block locally, send it.
        String datanode = chooseDatanode(lb.getLocations());
        computeMetadataAndSendReconstructedBlock(datanode, localBlockFile,
            lostBlock, blockContentsSize,
            lb.getDataProtocolVersion(), lb.getNamespaceID(), progress);
       
        numBlocksReconstructed++;

      } finally {
        localBlockFile.delete();
      }
      progress.progress();
    }
   
    LOG.info("Reconstructed " + numBlocksReconstructed + " blocks in " + srcPath);
    return true;
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.