Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.JobConf.addResource()


    } else {
      conf = new JobConf(ExecDriver.class);
    }

    if (jobConfFileName != null) {
      conf.addResource(new Path(jobConfFileName));
    }

    if (files != null) {
      conf.set("tmpfiles", files);
    }
View Full Code Here


        e.printStackTrace(System.err);
      }
    }

    if (jobConfFileName != null) {
      conf.addResource(new Path(jobConfFileName));
    }
    HiveConf hiveConf = new HiveConf(conf, BlockMergeTask.class);

    Log LOG = LogFactory.getLog(BlockMergeTask.class.getName());
    boolean isSilent = HiveConf.getBoolVar(conf,
View Full Code Here

        e.printStackTrace(System.err);
      }
    }

    if (jobConfFileName != null) {
      conf.addResource(new Path(jobConfFileName));
    }
    HiveConf hiveConf = new HiveConf(conf, PartialScanTask.class);

    Log LOG = LogFactory.getLog(PartialScanTask.class.getName());
    boolean isSilent = HiveConf.getBoolVar(conf,
View Full Code Here

              Integer.parseInt(nodeHttpPortString), appSubmitTime,
              Integer.parseInt(maxAppAttempts));
      ShutdownHookManager.get().addShutdownHook(
        new MRAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
      JobConf conf = new JobConf(new YarnConfiguration());
      conf.addResource(new Path(MRJobConfig.JOB_CONF_FILE));
     
      MRWebAppUtil.initialize(conf);
      String jobUserName = System
          .getenv(ApplicationConstants.Environment.USER.name());
      conf.set(MRJobConfig.USER_NAME, jobUserName);
View Full Code Here

      String historyPath =
          historyFilePath.substring(0,historyFilePath.lastIndexOf("_"));
      fs.copyToLocalFile(new Path(historyPath + "_conf.xml"),
                         new Path(tmpJHFolder.toString()));
      JobConf jobConf = new JobConf();
      jobConf.addResource(new Path(tmpJHFolder.toString()
                         + "/" + simulatedJobID + "_conf.xml"));
      jobConf.reloadConfiguration();
      return jobConf;

    }finally {
View Full Code Here

                                " If you plan to use local mode, please put -x local option in command line",
                                4010);
                }
                jc = new JobConf();
            }
            jc.addResource("pig-cluster-hadoop-site.xml");
            jc.addResource(YARN_SITE);
           
            // Trick to invoke static initializer of DistributedFileSystem to add hdfs-default.xml
            // into configuration
            new DistributedFileSystem();
View Full Code Here

                                4010);
                }
                jc = new JobConf();
            }
            jc.addResource("pig-cluster-hadoop-site.xml");
            jc.addResource(YARN_SITE);
           
            // Trick to invoke static initializer of DistributedFileSystem to add hdfs-default.xml
            // into configuration
            new DistributedFileSystem();
           
View Full Code Here

            properties.setProperty(JOB_TRACKER_LOCATION, LOCAL );
            properties.setProperty(FILE_SYSTEM_LOCATION, "file:///");
            properties.setProperty(ALTERNATIVE_FILE_SYSTEM_LOCATION, "file:///");

            jc = new JobConf(false);
            jc.addResource("core-default.xml");
            jc.addResource("mapred-default.xml");
            jc.addResource("yarn-default.xml");
            recomputeProperties(jc, properties);
        }
View Full Code Here

            properties.setProperty(FILE_SYSTEM_LOCATION, "file:///");
            properties.setProperty(ALTERNATIVE_FILE_SYSTEM_LOCATION, "file:///");

            jc = new JobConf(false);
            jc.addResource("core-default.xml");
            jc.addResource("mapred-default.xml");
            jc.addResource("yarn-default.xml");
            recomputeProperties(jc, properties);
        }

        cluster = jc.get(JOB_TRACKER_LOCATION);
View Full Code Here

            properties.setProperty(ALTERNATIVE_FILE_SYSTEM_LOCATION, "file:///");

            jc = new JobConf(false);
            jc.addResource("core-default.xml");
            jc.addResource("mapred-default.xml");
            jc.addResource("yarn-default.xml");
            recomputeProperties(jc, properties);
        }

        cluster = jc.get(JOB_TRACKER_LOCATION);
        nameNode = jc.get(FILE_SYSTEM_LOCATION);
View Full Code Here

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.