Examples of org.apache.hadoop.mapreduce.Job.addFileToClassPath()

org.apache.hadoop.mapreduce.Job.addFileToClassPath()
Add an file path to the current set of classpath entries It adds the file to cache as well. Files added with this method will not be unpacked while being added to the classpath. To add archives to classpath, use the {@link #addArchiveToClassPath(Path)}method instead. @param file Path of the file to be added

        SleepJob sleepJob = new SleepJob();
        sleepJob.setConf(mrCluster.getConfig());
        Job job = sleepJob.createJob(3, 0, 10000, 1, 0, 0);
        // //Job with reduces
        // Job job = sleepJob.createJob(3, 2, 10000, 1, 10000, 1);
        job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
        job.waitForCompletion(true);
        Assert.assertEquals(JobStatus.State.SUCCEEDED, job.getJobState());
        return null;
      }
    });

View Full Code Here


    FileInputFormat.setInputPaths(job, first);
    // Creates the Job Configuration
    job.addCacheFile(
        new URI(first.toUri().toString() + "#distributed.first.symlink"));
    job.addFileToClassPath(second);
    job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
    job.addArchiveToClassPath(third);
    job.addCacheArchive(fourth.toUri());
    job.createSymlink();
    job.setMaxMapAttempts(1); // speed up failures

View Full Code Here

    FileInputFormat.setInputPaths(job, first);
    // Creates the Job Configuration
    job.addCacheFile(
        new URI(first.toUri().toString() + "#distributed.first.symlink"));
    job.addFileToClassPath(second);
    job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
    job.addArchiveToClassPath(third);
    job.addCacheArchive(fourth.toUri());
    job.createSymlink();
    job.setMaxMapAttempts(1); // speed up failures

View Full Code Here

    FileSystem outFs = outDir.getFileSystem(conf);
    
    // Launch job with default option for temp dir. 
    // i.e. temp dir is ./tmp 
    Job job = new Job(conf);
    job.addFileToClassPath(APP_JAR);
    job.setJarByClass(TestMiniMRChildTask.class);
    job.setMaxMapAttempts(1); // speed up failures
    job.waitForCompletion(true);
    boolean succeeded = job.waitForCompletion(true);
    assertTrue(succeeded);

View Full Code Here

    conf.set("path", System.getenv("PATH"));
    conf.set(mapTaskJavaOptsKey, mapTaskJavaOpts);
    conf.set(reduceTaskJavaOptsKey, reduceTaskJavaOpts);


    Job job = new Job(conf);
    job.addFileToClassPath(APP_JAR);
    job.setJarByClass(TestMiniMRChildTask.class);
    job.setMaxMapAttempts(1); // speed up failures
    job.waitForCompletion(true);
    boolean succeeded = job.waitForCompletion(true);
    assertTrue("The environment checker job failed.", succeeded);

View Full Code Here

    fs.copyFromLocalFile(appMasterJar, appJar);
    fs.setPermission(appJar, new FsPermission("744"));


    Job job = Job.getInstance(conf);


    job.addFileToClassPath(appJar);
    String callerJar = JarFinder.getJar(caller);
    job.setJar(callerJar);


    MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(caller
        .getName(), noOfNMs);

View Full Code Here

    fs.copyFromLocalFile(appMasterJar, appJar);
    fs.setPermission(appJar, new FsPermission("744"));


    Job job = Job.getInstance(conf);


    job.addFileToClassPath(appJar);


    Path callerJar = new Path(JarFinder.getJar(caller));
    Path remoteCallerJar = new Path(testRootDir, callerJar.getName());
    fs.copyFromLocalFile(callerJar, remoteCallerJar);
    fs.setPermission(remoteCallerJar, new FsPermission("744"));

View Full Code Here


    Path callerJar = new Path(JarFinder.getJar(caller));
    Path remoteCallerJar = new Path(testRootDir, callerJar.getName());
    fs.copyFromLocalFile(callerJar, remoteCallerJar);
    fs.setPermission(remoteCallerJar, new FsPermission("744"));
    job.addFileToClassPath(remoteCallerJar);


    MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(caller
        .getName(), noOfNMs);
    job.getConfiguration().set("minimrclientcluster.caller.name",
        caller.getName());

View Full Code Here

    job.setOutputFormatClass(NullOutputFormat.class);
    FileInputFormat.setInputPaths(job, first);
    // Creates the Job Configuration
    job.addCacheFile(
      new URI(first.toUri().toString() + "#distributed.first.symlink"));
    job.addFileToClassPath(second);
    job.addArchiveToClassPath(third);
    job.addCacheArchive(fourth.toUri());
    
    // don't create symlink for LocalJobRunner
    if (!"local".equals(conf.get(JTConfig.JT_IPC_ADDRESS))) {

View Full Code Here

    job.setOutputFormatClass(NullOutputFormat.class);
    FileInputFormat.setInputPaths(job, first);
    // Creates the Job Configuration
    job.addCacheFile(
      new URI(first.toUri().toString() + "#distributed.first.symlink"));
    job.addFileToClassPath(second);
    job.addArchiveToClassPath(third);
    job.addCacheArchive(fourth.toUri());
    job.createSymlink();
    job.setMaxMapAttempts(1); // speed up failures

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.