Examples of FSDownload

org.apache.hadoop.yarn.util.FSDownload
Download a single URL to the local disk.

Examples of org.apache.hadoop.yarn.util.FSDownload

          LocalResource resource = request.getResource().getRequest();
          try {
            Path publicDirDestPath = dirsHandler.getLocalPathForWrite(
                "." + Path.SEPARATOR + ContainerLocalizer.FILECACHE,
                ContainerLocalizer.getEstimatedSize(resource), true);
            pending.put(queue.submit(new FSDownload(
                lfs, null, conf, publicDirDestPath, resource, new Random())),
                request);
            attempts.put(key, new LinkedList<LocalizerResourceRequestEvent>());
          } catch (IOException e) {
            LOG.error("Local path for public localization is not found. "

View Full Code Here

Examples of org.apache.hadoop.yarn.util.FSDownload

      .setNameFormat("ContainerLocalizer Downloader").build());
  }


  Callable<Path> download(LocalDirAllocator lda, LocalResource rsrc,
      UserGroupInformation ugi) {
    return new FSDownload(lfs, ugi, conf, lda, rsrc, new Random());
  }

View Full Code Here

Examples of org.apache.hadoop.yarn.util.FSDownload

      LocalResourceRequest key = request.getResource().getRequest();
      LOG.info("Downloading public rsrc:" + key);
      synchronized (attempts) {
        List<LocalizerResourceRequestEvent> sigh = attempts.get(key);
        if (null == sigh) {
          pending.put(queue.submit(new FSDownload(
                  lfs, null, conf, publicDirs,
                  request.getResource().getRequest(), new Random())),
              request);
          attempts.put(key, new LinkedList<LocalizerResourceRequestEvent>());
        } else {

View Full Code Here

Examples of org.apache.hadoop.yarn.util.FSDownload

    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    
    Map<LocalResource, Future<Path>> resourcesToPaths = Maps.newHashMap();
    ExecutorService exec = Executors.newCachedThreadPool();
    for (LocalResource resource : localResources.values()) {
      Callable<Path> download = new FSDownload(localFSFileContext, ugi, conf,
          localDirAllocator, resource, new Random());
      Future<Path> future = exec.submit(download);
      resourcesToPaths.put(resource, future);
    }
    for (LocalResource resource : localResources.values()) {

View Full Code Here

Examples of org.apache.hadoop.yarn.util.FSDownload

          LocalResource resource = request.getResource().getRequest();
          try {
            Path publicDirDestPath = dirsHandler.getLocalPathForWrite(
                "." + Path.SEPARATOR + ContainerLocalizer.FILECACHE,
                ContainerLocalizer.getEstimatedSize(resource), true);
            pending.put(queue.submit(new FSDownload(
                lfs, null, conf, publicDirDestPath, resource, new Random())),
                request);
            attempts.put(key, new LinkedList<LocalizerResourceRequestEvent>());
          } catch (IOException e) {
            LOG.error("Local path for public localization is not found. "

View Full Code Here

Examples of org.apache.hadoop.yarn.util.FSDownload

      exec = Executors.newCachedThreadPool(tf);
      Path destPath = localDirAllocator.getLocalPathForWrite(".", conf);
      Map<LocalResource, Future<Path>> resourcesToPaths = Maps.newHashMap();
      for (LocalResource resource : localResources.values()) {
        Callable<Path> download =
            new FSDownload(localFSFileContext, ugi, conf, new Path(destPath,
                Long.toString(uniqueNumberGenerator.incrementAndGet())),
                resource);
        Future<Path> future = exec.submit(download);
        resourcesToPaths.put(resource, future);
      }

View Full Code Here

Examples of org.apache.hadoop.yarn.util.FSDownload

                new Path(publicDirDestPath, Long.toString(publicRsrc
                  .nextUniqueNumber()));
            // explicitly synchronize pending here to avoid future task
            // completing and being dequeued before pending updated
            synchronized (pending) {
              pending.put(queue.submit(new FSDownload(lfs, null, conf,
                  publicDirDestPath, resource, request.getContext().getStatCache())),
                  request);
            }
          } catch (IOException e) {
            rsrc.unlock();

View Full Code Here

Examples of org.apache.hadoop.yarn.util.FSDownload

  }


  Callable<Path> download(Path path, LocalResource rsrc,
      UserGroupInformation ugi) throws IOException {
    DiskChecker.checkDir(new File(path.toUri().getRawPath()));
    return new FSDownload(lfs, ugi, conf, path, rsrc);
  }

View Full Code Here

Examples of org.apache.hadoop.yarn.util.FSDownload

          LocalResource resource = request.getResource().getRequest();
          try {
            Path publicDirDestPath = dirsHandler.getLocalPathForWrite(
                "." + Path.SEPARATOR + ContainerLocalizer.FILECACHE,
                ContainerLocalizer.getEstimatedSize(resource), true);
            pending.put(queue.submit(new FSDownload(
                lfs, null, conf, publicDirDestPath, resource, new Random())),
                request);
            attempts.put(key, new LinkedList<LocalizerResourceRequestEvent>());
          } catch (IOException e) {
            LOG.error("Local path for public localization is not found. "

View Full Code Here

Examples of org.apache.hadoop.yarn.util.FSDownload

  }


  Callable<Path> download(LocalDirAllocator lda, LocalResource rsrc,
      UserGroupInformation ugi) throws IOException {
    Path destPath = lda.getLocalPathForWrite(".", getEstimatedSize(rsrc), conf);
    return new FSDownload(lfs, ugi, conf, destPath, rsrc, new Random());
  }

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.