Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.JobConf.addResource()

109
110
111
112
113
114
115
116
117
118
119
    public JobConf getLocalConf(Properties properties) {
        JobConf jc = new JobConf(false);

        jc.addResource(MAPRED_DEFAULT_SITE);
        jc.addResource(YARN_DEFAULT_SITE);
        jc.addResource(CORE_DEFAULT_SITE);

        return jc;
    }
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
    public JobConf getLocalConf(Properties properties) {
        JobConf jc = new JobConf(false);

        jc.addResource(MAPRED_DEFAULT_SITE);
        jc.addResource(YARN_DEFAULT_SITE);
        jc.addResource(CORE_DEFAULT_SITE);

        return jc;
    }

    public JobConf getExecConf(Properties properties) throws ExecException {
View Full Code Here
138
139
140
141
142
143
144
145
146
147
148
                                + " If you plan to use local mode, please put -x local option in command line",
                                4010);
            }
            jc = new JobConf();
        }
        jc.addResource("pig-cluster-hadoop-site.xml");
        jc.addResource(YARN_SITE);
        return jc;
    }

    private void init(Properties properties) throws ExecException {
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
                                4010);
            }
            jc = new JobConf();
        }
        jc.addResource("pig-cluster-hadoop-site.xml");
        jc.addResource(YARN_SITE);
        return jc;
    }

    private void init(Properties properties) throws ExecException {
        String cluster = null;
View Full Code Here
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
              Integer.parseInt(nodeHttpPortString), appSubmitTime,
              Integer.parseInt(maxAppAttempts));
      ShutdownHookManager.get().addShutdownHook(
        new MRAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
      JobConf conf = new JobConf(new YarnConfiguration());
      conf.addResource(new Path(MRJobConfig.JOB_CONF_FILE));
      String jobUserName = System
          .getenv(ApplicationConstants.Environment.USER.name());
      conf.set(MRJobConfig.USER_NAME, jobUserName);
      // Do not automatically close FileSystem objects so that in case of
      // SIGTERM I have a chance to write out the job history. I'll be closing
View Full Code Here
348
349
350
351
352
353
354
355
356
357
358
      fs = jhpath.getFileSystem(conf);
      fs.copyToLocalFile(jhpath,new Path(tmpJHFolder.toString()));
      fs.copyToLocalFile(new Path(jhpath.getParent() + "/" + simulatedJobID + "_conf.xml"),
                         new Path(tmpJHFolder.toString()));
      JobConf jobConf = new JobConf();
      jobConf.addResource(new Path(tmpJHFolder.toString()
                         + "/" + simulatedJobID + "_conf.xml"));
      jobConf.reloadConfiguration();
      return jobConf;

    }finally {
View Full Code Here
166
167
168
169
170
171
172
173
174
175
176
      jobName = args[5];
    }
   
    JobConf theJob = new JobConf(conf);
    if (specFile != null) {
      theJob.addResource(specFile);
    }
    String userJarFile = theJob.get("user.jar.file");
    if (userJarFile == null) {
      theJob.setJarByClass(caller != null ? caller : ValueAggregatorJob.class);
    } else {
View Full Code Here
72
73
74
75
76
77
78
79
80
81
82
   * @param configuration The Hadoop configuration against which the Copy-mapper must run.
   * @throws Exception, on failure.
   */
  public DistCp(Configuration configuration, DistCpOptions inputOptions) throws Exception {
    final JobConf config = new JobConf(configuration);
    config.addResource(DISTCP_DEFAULT_XML);
    setConf(config);
    this.inputOptions = inputOptions;
    this.metaFolder   = createMetaFolderPath();
  }

View Full Code Here
381
382
383
384
385
386
387
388
389
390
391
   * object
   * @return Configuration which includes properties from distcp-default.xml
   */
  private static JobConf getDefaultConf() {
    JobConf config = new JobConf();
    config.addResource(DISTCP_DEFAULT_XML);
    return config;
  }

  private synchronized void cleanup() {
    try {
View Full Code Here
154
155
156
157
158
159
160
161
162
163
164
                        "If you plan to use local mode, please put -x local option in command line",
                        4010);
            }

            jc = new JobConf();
            jc.addResource("pig-cluster-hadoop-site.xml");
            jc.addResource(YARN_SITE);
           
            // Trick to invoke static initializer of DistributedFileSystem to add hdfs-default.xml
            // into configuration
            new DistributedFileSystem();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.