Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.ClusterStatus

473
474
475
476
477
478
479
480
481
482
483
484
      // amount of data per map.
      int nFiles = srcPaths.length;
      int numMaps = nFiles;
      if (numMaps > MAX_NUM_MAPS) { numMaps = MAX_NUM_MAPS; }
      JobClient client = new JobClient(jobConf);
      ClusterStatus cluster = client.getClusterStatus();
      int tmpMaps = cluster.getTaskTrackers() * MAX_MAPS_PER_NODE;
      if (numMaps > tmpMaps) { numMaps = tmpMaps; }
      if (numMaps == 0) { numMaps = 1; }
      jobConf.setNumMapTasks(numMaps);
     
      jobConf.setBoolean(readFailuresAttribute, ignoreReadFailures);
View Full Code Here
169
170
171
172
173
174
175
176
177
178
179
180
   
    jobConf.setMapperClass(Map.class);       
    jobConf.setReducerClass(NNBench.class);
   
    JobClient client = new JobClient(jobConf);
    ClusterStatus cluster = client.getClusterStatus();
    int numMaps = cluster.getTaskTrackers() *
         jobConf.getInt("test.nnbench.maps_per_host", 10);
    jobConf.setNumMapTasks(numMaps);
    System.out.println("Running " + numMaps + " maps.");
    jobConf.setNumReduceTasks(1);
   
View Full Code Here
170
171
172
173
174
175
176
177
178
179
180
181
   
    jobConf.setMapperClass(Map.class);       
    jobConf.setReducerClass(RandomWriter.class);
   
    JobClient client = new JobClient(jobConf);
    ClusterStatus cluster = client.getClusterStatus();
    int numMaps = cluster.getTaskTrackers() *
         jobConf.getInt("test.randomwriter.maps_per_host", 10);
    jobConf.setNumMapTasks(numMaps);
    System.out.println("Running " + numMaps + " maps.");
    jobConf.setNumReduceTasks(1);
   
View Full Code Here
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
    }
   
    Path outDir = new Path(args[0]);
    Configuration conf = getConf();
    JobClient client = new JobClient(conf);
    ClusterStatus cluster = client.getClusterStatus();
    int numMapsPerHost = conf.getInt(MAPS_PER_HOST, 10);
    long numBytesToWritePerMap = conf.getLong(BYTES_PER_MAP,
                                             1*1024*1024*1024);
    if (numBytesToWritePerMap == 0) {
      System.err.println("Cannot have" + BYTES_PER_MAP + " set to 0");
      return -2;
    }
    long totalBytesToWrite = conf.getLong(TOTAL_BYTES,
         numMapsPerHost*numBytesToWritePerMap*cluster.getTaskTrackers());
    int numMaps = (int) (totalBytesToWrite / numBytesToWritePerMap);
    if (numMaps == 0 && totalBytesToWrite > 0) {
      numMaps = 1;
      conf.setLong(BYTES_PER_MAP, totalBytesToWrite);
    }
View Full Code Here
344
345
346
347
348
349
350
351
352
353
354
355
    if (numMaps > MAX_NUM_MAPS) { numMaps = MAX_NUM_MAPS; }
    if (numMaps > (int) (totalBytes / MIN_BYTES_PER_MAP)) {
        numMaps = (int) (totalBytes / MIN_BYTES_PER_MAP);
    }
    JobClient client = new JobClient(jobConf);
    ClusterStatus cluster = client.getClusterStatus();
    int tmpMaps = cluster.getTaskTrackers() * MAX_MAPS_PER_NODE;
    if (numMaps > tmpMaps) { numMaps = tmpMaps; }
    if (numMaps == 0) { numMaps = 1; }
    jobConf.setNumMapTasks(numMaps);
   
    for(int idx=0; idx < numMaps; ++idx) {
View Full Code Here
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
      throws IOException {
    // from RandomWriter
    job.setInputFormat(RandomInputFormat.class);
    job.setMapperClass(RandomMapOutput.class);

    final ClusterStatus cluster = new JobClient(job).getClusterStatus();
    int numMapsPerHost = job.getInt("test.randomtextwrite.maps_per_host", 10);
    long numBytesToWritePerMap =
      job.getLong("test.randomtextwrite.bytes_per_map", 1*1024*1024*1024);
    if (numBytesToWritePerMap == 0) {
      throw new IOException(
          "Cannot have test.randomtextwrite.bytes_per_map set to 0");
    }
    long totalBytesToWrite = job.getLong("test.randomtextwrite.total_bytes",
         numMapsPerHost * numBytesToWritePerMap * cluster.getTaskTrackers());
    int numMaps = (int)(totalBytesToWrite / numBytesToWritePerMap);
    if (numMaps == 0 && totalBytesToWrite > 0) {
      numMaps = 1;
      job.setLong("test.randomtextwrite.bytes_per_map", totalBytesToWrite);
    }
View Full Code Here
162
163
164
165
166
167
168
169
170
171
172
173
   
    jobConf.setMapperClass(Map.class);       
    jobConf.setReducerClass(RandomWriter.class);
   
    JobClient client = new JobClient(jobConf);
    ClusterStatus cluster = client.getClusterStatus();
    int numMaps = cluster.getTaskTrackers() *
         jobConf.getInt("test.randomwriter.maps_per_host", 10);
    jobConf.setNumMapTasks(numMaps);
    System.out.println("Running " + numMaps + " maps.");
    jobConf.setNumReduceTasks(1);
   
View Full Code Here
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
   
    job.setInputFormat(RandomWriter.RandomInputFormat.class);
    job.setMapperClass(Map.class);       
   
    JobClient client = new JobClient(job);
    ClusterStatus cluster = client.getClusterStatus();
    int numMapsPerHost = job.getInt("test.randomtextwrite.maps_per_host", 10);
    long numBytesToWritePerMap = job.getLong("test.randomtextwrite.bytes_per_map",
                                             1*1024*1024*1024);
    if (numBytesToWritePerMap == 0) {
      System.err.println("Cannot have test.randomtextwrite.bytes_per_map set to 0");
      return -2;
    }
    long totalBytesToWrite = job.getLong("test.randomtextwrite.total_bytes",
         numMapsPerHost*numBytesToWritePerMap*cluster.getTaskTrackers());
    int numMaps = (int) (totalBytesToWrite / numBytesToWritePerMap);
    if (numMaps == 0 && totalBytesToWrite > 0) {
      numMaps = 1;
      job.setLong("test.randomtextwrite.bytes_per_map", totalBytesToWrite);
    }
View Full Code Here
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
  /**
   * Return the status information about the Map-Reduce cluster
   */
  public ClusterStatus getClusterStatus() throws Exception {
    ClusterStatus cs;
    try {
      JobConf job = new JobConf(conf);
      JobClient jc = new JobClient(job);
      cs = jc.getClusterStatus();
    } catch (Exception e) {
      e.printStackTrace();
      throw e;
    }
    LOG.info("Returning cluster status: " + cs.toString());
    return cs;
  }
View Full Code Here
206
207
208
209
210
211
212
213
214
215
216
217
  }

  private void waitForTaskTrackers(JobClient client) throws IOException {
    LOG.info("Waiting for tasktrackers...");
    while (true) {
      ClusterStatus clusterStatus = client.getClusterStatus();
      int taskTrackerCount = clusterStatus.getTaskTrackers();
      if (taskTrackerCount > 0) {
        LOG.info("{} tasktrackers reported in. Continuing.", taskTrackerCount);
        break;
      }
      try {
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.ClusterStatus

  • com.google.HadoopMonitor$ProgressResult
  • com.nexr.rhive.hadoop.JobManager
  • org.apache.accumulo.server.monitor.servlets.DefaultServlet
  • org.apache.accumulo.server.test.continuous.ContinuousStatsCollector
  • org.apache.accumulo.test.continuous.ContinuousStatsCollector
  • org.apache.hadoop.examples.Join
  • org.apache.hadoop.examples.NNBench
  • org.apache.hadoop.examples.RandomTextWriter
  • org.apache.hadoop.examples.RandomWriter
  • org.apache.hadoop.examples.Sort
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.