Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.Reporter

86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
      writer.process(form);
      reporter.progress();
    }

    // close the shard
    final Reporter fReporter = reporter;
    new Closeable() {
      boolean closed = false;

      public void close() throws IOException {
        // spawn a thread to give progress heartbeats
        Thread prog = new Thread() {
          public void run() {
            while (!closed) {
              try {
                fReporter.setStatus("closing");
                Thread.sleep(1000);
              } catch (InterruptedException e) {
                continue;
              } catch (Throwable e) {
                return;
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
    LOG.info("testCopyFromHostBogusHeader");
    JobConf job = new JobConf();
    TaskAttemptID id = TaskAttemptID.forName("attempt_0_1_r_1_1");
    ShuffleScheduler<Text, Text> ss = mock(ShuffleScheduler.class);
    MergeManager<Text, Text> mm = mock(MergeManager.class);
    Reporter r = mock(Reporter.class);
    ShuffleClientMetrics metrics = mock(ShuffleClientMetrics.class);
    ExceptionReporter except = mock(ExceptionReporter.class);
    SecretKey key = JobTokenSecretManager.createSecretKey(new byte[]{0,0,0,0});
    HttpURLConnection connection = mock(HttpURLConnection.class);
   
    Counters.Counter allErrs = mock(Counters.Counter.class);
    when(r.getCounter(anyString(), anyString()))
      .thenReturn(allErrs);
   
    Fetcher<Text,Text> underTest = new FakeFetcher<Text,Text>(job, id, ss, mm,
        r, metrics, except, key, connection);
   
View Full Code Here
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
    LOG.info("testCopyFromHostWait");
    JobConf job = new JobConf();
    TaskAttemptID id = TaskAttemptID.forName("attempt_0_1_r_1_1");
    ShuffleScheduler<Text, Text> ss = mock(ShuffleScheduler.class);
    MergeManager<Text, Text> mm = mock(MergeManager.class);
    Reporter r = mock(Reporter.class);
    ShuffleClientMetrics metrics = mock(ShuffleClientMetrics.class);
    ExceptionReporter except = mock(ExceptionReporter.class);
    SecretKey key = JobTokenSecretManager.createSecretKey(new byte[]{0,0,0,0});
    HttpURLConnection connection = mock(HttpURLConnection.class);
   
    Counters.Counter allErrs = mock(Counters.Counter.class);
    when(r.getCounter(anyString(), anyString()))
      .thenReturn(allErrs);
   
    Fetcher<Text,Text> underTest = new FakeFetcher<Text,Text>(job, id, ss, mm,
        r, metrics, except, key, connection);
   
View Full Code Here
364
365
366
367
368
369
370
371
372
373
374
    // CHECKSTYLE: resume LineLengthCheck
    HadoopUtils.setWorkOutputDir(taskAttemptContext);
    JobConf jobConf = new JobConf(taskAttemptContext.getConfiguration());
    int fileId = CREATED_FILES_COUNTER.incrementAndGet();
    String name = FileOutputFormat.getUniqueName(jobConf, "part-" + fileId);
    Reporter reporter = new ProgressReporter(taskAttemptContext);
    org.apache.hadoop.mapred.RecordWriter<WritableComparable, Writable> baseWriter =
        baseOutputFormat.getRecordWriter(null, jobConf, name, reporter);
    LOG.info("getBaseRecordWriter: Created new {} with file {}", baseWriter, name);
    return baseWriter;
  }
View Full Code Here
498
499
500
501
502
503
504
505
506
507
508
    // CHECKSTYLE: resume LineLengthCheck
    HadoopUtils.setWorkOutputDir(taskAttemptContext);
    JobConf jobConf = new JobConf(taskAttemptContext.getConfiguration());
    int fileId = CREATED_FILES_COUNTER.incrementAndGet();
    String name = FileOutputFormat.getUniqueName(jobConf, "part-" + fileId);
    Reporter reporter = new ProgressReporter(taskAttemptContext);
    org.apache.hadoop.mapred.RecordWriter<WritableComparable, Writable> baseWriter =
        baseOutputFormat.getRecordWriter(null, jobConf, name, reporter);
    LOG.info("getBaseRecordWriter: Created new {} with file {}", baseWriter, name);
    return baseWriter;
  }
View Full Code Here
405
406
407
408
409
410
411
412
413
414
415
    // CHECKSTYLE: resume LineLengthCheck
    HadoopUtils.setWorkOutputDir(taskAttemptContext);
    JobConf jobConf = new JobConf(taskAttemptContext.getConfiguration());
    int fileId = CREATED_FILES_COUNTER.incrementAndGet();
    String name = FileOutputFormat.getUniqueName(jobConf, "part-" + fileId);
    Reporter reporter = new ProgressReporter(taskAttemptContext);
    org.apache.hadoop.mapred.RecordWriter<WritableComparable, Writable> baseWriter =
        baseOutputFormat.getRecordWriter(null, jobConf, name, reporter);
    LOG.info("getBaseRecordWriter: Created new {} with file {}", baseWriter, name);
    return baseWriter;
  }
View Full Code Here
26
27
28
29
30
31
32
33
34
35
36
37
                        Properties tableProperties,
                        Progressable progress) throws IOException {
    final RecordWriter result =
      super.getHiveRecordWriter(jc,outPath,valueClass,isCompressed,
        tableProperties,progress);
    final Reporter reporter = (Reporter) progress;
    reporter.setStatus("got here");
    System.out.println("Got a reporter " + reporter);
    return new RecordWriter() {
      @Override
      public void write(Writable w) throws IOException {
        if (w instanceof Text) {
View Full Code Here
403
404
405
406
407
408
409
410
411
412
413
    // CHECKSTYLE: resume LineLengthCheck
    HadoopUtils.setWorkOutputDir(taskAttemptContext);
    JobConf jobConf = new JobConf(taskAttemptContext.getConfiguration());
    int fileId = CREATED_FILES_COUNTER.incrementAndGet();
    String name = FileOutputFormat.getUniqueName(jobConf, "part-" + fileId);
    Reporter reporter = new ProgressReporter(taskAttemptContext);
    org.apache.hadoop.mapred.RecordWriter<WritableComparable, Writable> baseWriter =
        baseOutputFormat.getRecordWriter(null, jobConf, name, reporter);
    LOG.info("getBaseRecordWriter: Created new {} with file {}", baseWriter, name);
    return baseWriter;
  }
View Full Code Here
512
513
514
515
516
517
518
519
520
521
522
    // CHECKSTYLE: resume LineLengthCheck
    HadoopUtils.setWorkOutputDir(taskAttemptContext);
    JobConf jobConf = new JobConf(taskAttemptContext.getConfiguration());
    int fileId = CREATED_FILES_COUNTER.incrementAndGet();
    String name = FileOutputFormat.getUniqueName(jobConf, "part-" + fileId);
    Reporter reporter = new ProgressReporter(taskAttemptContext);
    org.apache.hadoop.mapred.RecordWriter<WritableComparable, Writable> baseWriter =
        baseOutputFormat.getRecordWriter(null, jobConf, name, reporter);
    LOG.info("getBaseRecordWriter: Created new {} with file {}", baseWriter, name);
    return baseWriter;
  }
View Full Code Here
132
133
134
135
136
137
138
139
   * @throws IOException I/O errors
   */
  public org.apache.hadoop.mapred.RecordReader<WritableComparable, Writable>
  getBaseRecordReader(JobConf jobConf, Progressable progressable)
    throws IOException {
    Reporter reporter = new ProgressReporter(progressable);
    return baseInputFormat.getRecordReader(baseSplit, jobConf, reporter);
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.Reporter

  • co.nubetech.hiho.mapreduce.TestDBInputAvroMapper
  • com.facebook.giraph.hive.impl.input.HiveApiInputSplit
  • com.facebook.giraph.hive.input.HInputSplit
  • com.facebook.giraph.hive.output.HiveApiOutputFormat
  • com.facebook.hiveio.input.HInputSplit
  • com.facebook.hiveio.output.HiveApiOutputFormat
  • com.senseidb.indexing.hadoop.reduce.SenseiReducer
  • com.twitter.elephantbird.mapreduce.input.MapReduceInputFormatWrapper$RecordReaderWrapper
  • org.apache.hadoop.contrib.index.mapred.IndexUpdateReducer
  • org.apache.hadoop.hbase.mapred.TestGroupingTableMap
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.