Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.JobConf.addResource()

481
482
483
484
485
486
487
488
489
490
491
      JobClient.runJob(job);
      System.out.println("Processed TaskTracker ClientTrace.");

      // Process DataNode clienttrace entries
      job = new JobConf(new ChukwaConfiguration(), FSMBuilder.class);
      job.addResource(System.getenv("CHUKWA_CONF_DIR")+File.separator+"chukwa-demux-conf.xml");
      job.setJobName("Chukwa-FSMBuilder_" + day.format(new Date()));
      job.setMapperClass(DataNodeClientTraceMapper.class);
      job.setPartitionerClass(FSMIntermedEntryPartitioner.class);
      job.setReducerClass(FSMBuilder.FSMReducer.class);
      job.setMapOutputValueClass(FSMIntermedEntry.class);
View Full Code Here
152
153
154
155
156
157
158
159
160
161
162
        if( this.pigContext.getExecType() == ExecType.LOCAL ) {
            // We dont load any configurations here
            jc = new JobConf( false );
        } else {
            jc = new JobConf();
            jc.addResource("pig-cluster-hadoop-site.xml");
        }
           
        //the method below alters the properties object by overriding the
        //hadoop properties with the values from properties and recomputing
        //the properties
View Full Code Here
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
              Integer.parseInt(nodeHttpPortString), appSubmitTime,
              Integer.parseInt(maxAppAttempts));
      ShutdownHookManager.get().addShutdownHook(
        new MRAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
      JobConf conf = new JobConf(new YarnConfiguration());
      conf.addResource(new Path(MRJobConfig.JOB_CONF_FILE));
      String jobUserName = System
          .getenv(ApplicationConstants.Environment.USER.name());
      conf.set(MRJobConfig.USER_NAME, jobUserName);
      // Do not automatically close FileSystem objects so that in case of
      // SIGTERM I have a chance to write out the job history. I'll be closing
View Full Code Here
323
324
325
326
327
328
329
330
331
332
333
        e.printStackTrace(System.err);
      }
    }

    if (jobConfFileName != null) {
      conf.addResource(new Path(jobConfFileName));
    }
    HiveConf hiveConf = new HiveConf(conf, PartialScanTask.class);

    Log LOG = LogFactory.getLog(PartialScanTask.class.getName());
    boolean isSilent = HiveConf.getBoolVar(conf,
View Full Code Here
318
319
320
321
322
323
324
325
326
327
328
        e.printStackTrace(System.err);
      }
    }

    if (jobConfFileName != null) {
      conf.addResource(new Path(jobConfFileName));
    }
    HiveConf hiveConf = new HiveConf(conf, BlockMergeTask.class);

    Log LOG = LogFactory.getLog(BlockMergeTask.class.getName());
    boolean isSilent = HiveConf.getBoolVar(conf,
View Full Code Here
640
641
642
643
644
645
646
647
648
649
650
    } else {
      conf = new JobConf(ExecDriver.class);
    }

    if (jobConfFileName != null) {
      conf.addResource(new Path(jobConfFileName));
    }

    if (files != null) {
      conf.set("tmpfiles", files);
    }
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
      jobName = args[5];
    }
   
    JobConf theJob = new JobConf(conf);
    if (specFile != null) {
      theJob.addResource(specFile);
    }
    String userJarFile = theJob.get("user.jar.file");
    if (userJarFile == null) {
      theJob.setJarByClass(ValueAggregator.class);
    } else {
View Full Code Here
339
340
341
342
343
344
345
346
347
348
349
    Parser parser = cli.createParser();
    try {
      CommandLine results = parser.parse(args);
      JobConf conf = new JobConf();
      if (results.hasOption("-conf")) {
        conf.addResource(new Path((String) results.getValue("-conf")));
      }
      if (results.hasOption("-input")) {
        conf.setInputPath(new Path((String) results.getValue("-input")));
      }
      if (results.hasOption("-output")) {
View Full Code Here
148
149
150
151
152
153
154
155
156
157
158
      jobName = args[5];
    }
   
    JobConf theJob = new JobConf(conf);
    if (specFile != null) {
      theJob.addResource(specFile);
    }
    String userJarFile = theJob.get("user.jar.file");
    if (userJarFile == null) {
      theJob.setJarByClass(ValueAggregator.class);
    } else {
View Full Code Here
348
349
350
351
352
353
354
355
356
357
358
    Parser parser = cli.createParser();
    try {
      CommandLine results = parser.parse(args);
      JobConf conf = new JobConf();
      if (results.hasOption("-conf")) {
        conf.addResource(new Path((String) results.getValue("-conf")));
      }
      if (results.hasOption("-input")) {
        conf.setInputPath(new Path((String) results.getValue("-input")));
      }
      if (results.hasOption("-output")) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.