Examples of org.apache.twill.filesystem.Location.toURI()

Class org.apache.twill.filesystem.Location

Examples of org.apache.twill.filesystem.Location.toURI()

org.apache.twill.filesystem.Location.toURI()
@return A {@link java.net.URI} for this location.

    // Create a jar file with by tracing dependency
    ApplicationBundler bundler = new ApplicationBundler(ImmutableList.<String>of());
    bundler.createBundle(location, ApplicationBundler.class);


    File targetDir = tmpDir.newFolder();
    unjar(new File(location.toURI()), targetDir);


    // Load the class back, it should be loaded by the custom classloader
    ClassLoader classLoader = createClassLoader(targetDir);
    Class<?> clz = classLoader.loadClass(ApplicationBundler.class.getName());
    Assert.assertSame(classLoader, clz.getClassLoader());

View Full Code Here

      }


      UserGroupInformation.getCurrentUser().addCredentials(credentials);
      this.credentials = credentials;


      LOG.info("Secure store updated from {}.", location.toURI());


    } catch (Throwable t) {
      LOG.error("Failed to update secure store.", t);
    }

View Full Code Here

          location = copyFromURL(url, createTempLocation(Paths.appendSuffix(url.getFile(), localFile.getName())));
          LOG.debug("Done {} : {}", runnableName, url);
        }


        localFiles.put(runnableName,
                       new DefaultLocalFile(localFile.getName(), location.toURI(), location.lastModified(),
                                            location.length(), localFile.isArchive(), localFile.getPattern()));
      }
    }
    LOG.debug("Done Runnable LocalFiles");
  }

View Full Code Here

    }


    Location uploadDir = locationFactory.create(archiveDir).append("account_placeholder");
    String archiveName = name + ".jar";
    Location archive = uploadDir.append(archiveName);
    LOG.info("Storing module {} jar at {}", name, archive.toURI().toString());


    if (!uploadDir.exists() && !uploadDir.mkdirs()) {
      LOG.warn("Unable to create directory '{}'", uploadDir.getName());
    }

View Full Code Here

    try {
      try {
        Location programJarCopy = copyProgramJar();
        try {
          job.setJar(jobJar.toURI().toString());
          job.addFileToClassPath(new Path(programJarCopy.toURI()));


          MapReduceContextConfig contextConfig = new MapReduceContextConfig(job);
          // We start long-running tx to be used by mapreduce job tasks.
          Transaction tx = txClient.startLong();
          try {

View Full Code Here

        StreamBatchReadable stream = new StreamBatchReadable(URI.create(inputDataSetName));
        StreamConfig streamConfig = streamAdmin.getConfig(stream.getStreamName());
        Location streamPath = StreamUtils.createGenerationLocation(streamConfig.getLocation(),
                                                                   StreamUtils.getGeneration(streamConfig));


        LOG.info("Using Stream as input from {}", streamPath.toURI());


        TextStreamInputFormat.setTTL(job, streamConfig.getTTL());
        TextStreamInputFormat.setStreamPath(job, streamPath.toURI());
        TextStreamInputFormat.setTimeRange(job, stream.getStartTime(), stream.getEndTime());
        job.setInputFormatClass(TextStreamInputFormat.class);

View Full Code Here

                                                                   StreamUtils.getGeneration(streamConfig));


        LOG.info("Using Stream as input from {}", streamPath.toURI());


        TextStreamInputFormat.setTTL(job, streamConfig.getTTL());
        TextStreamInputFormat.setStreamPath(job, streamPath.toURI());
        TextStreamInputFormat.setTimeRange(job, stream.getStartTime(), stream.getEndTime());
        job.setInputFormatClass(TextStreamInputFormat.class);


      } else {
        // We checked on validation phase that it implements BatchReadable

View Full Code Here

      locationFactory.create(String.format("%s.%s.%s.%s.%s.jar",
                                           ProgramType.MAPREDUCE.name().toLowerCase(),
                                           programId.getAccountId(), programId.getApplicationId(),
                                           programId.getId(), context.getRunId().getId()));


    LOG.debug("Creating Job jar: {}", jobJar.toURI());


    Set<Class<?>> classes = Sets.newHashSet();
    classes.add(MapReduce.class);
    classes.add(DataSetOutputFormat.class);
    classes.add(DataSetInputFormat.class);

View Full Code Here

    ClassLoader oldCLassLoader = Thread.currentThread().getContextClassLoader();
    Thread.currentThread().setContextClassLoader(jobConf.getConfiguration().getClassLoader());
    appBundler.createBundle(jobJar, classes);
    Thread.currentThread().setContextClassLoader(oldCLassLoader);


    LOG.info("Built MapReduce Job Jar at {}", jobJar.toURI());
    return jobJar;
  }


  /**
   * Creates a temp copy of the program jar.

View Full Code Here

                    ProgramType.MAPREDUCE.name().toLowerCase(),
                    programId.getAccountId(), programId.getApplicationId(),
                    programId.getId(), context.getRunId().getId()));


    ByteStreams.copy(Locations.newInputSupplier(programJarLocation), Locations.newOutputSupplier(programJarCopy));
    LOG.info("Copied Program Jar to {}, source: {}", programJarCopy.toURI(), programJarLocation.toURI());
    return programJarCopy;
  }


  private Runnable createCleanupTask(final Location... locations) {
    return new Runnable() {

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.