Examples of org.apache.hadoop.mapred.RecordWriter

org.apache.hadoop.mapred.RecordWriter
RecordWriter writes the output <key, value> pairs to an output file.
RecordWriter implementations write the job outputs to the {@link FileSystem}. @see OutputFormat

      checkTokenName(multiName);
    }


    String baseFileName = (multi) ? namedOutput + "_" + multiName : namedOutput;


    final RecordWriter writer =
      getRecordWriter(namedOutput, baseFileName, reporter);


    return new AvroCollector() {
   
      @SuppressWarnings({"unchecked"})
      public void collect(Object key) throws IOException{
       AvroWrapper wrapper = new AvroWrapper(key);
       writer.write(wrapper, NullWritable.get());
      }
      
      public void collect(Object key,Object value) throws IOException
      {
        writer.write(key,value);
      }  
    
    };
  }

View Full Code Here

          ObjectInspectorFactory.getReflectionObjectInspector(NestedRow.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
    SerDe serde = new OrcSerde();
    OutputFormat<?, ?> outFormat = new OrcOutputFormat();
    RecordWriter writer =
        outFormat.getRecordWriter(fs, conf, testFilePath.toString(),
            Reporter.NULL);
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(1,2,3), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(4,5,6), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(7,8,9), inspector));
    writer.close(Reporter.NULL);
    serde = new OrcSerde();
    properties.setProperty("columns", "z,r");
    properties.setProperty("columns.types", "int:struct<x:int,y:int>");
    serde.initialize(conf, properties);
    inspector = (StructObjectInspector) serde.getObjectInspector();

View Full Code Here

  public static class SegmentOutputFormat extends OutputFormatBase {
    private static final String DEFAULT_SLICE = "default";
    
    @Override
    public RecordWriter getRecordWriter(final FileSystem fs, final JobConf job, final String name, final Progressable progress) throws IOException {
      return new RecordWriter() {
        MapFile.Writer c_out = null;
        MapFile.Writer f_out = null;
        MapFile.Writer pd_out = null;
        MapFile.Writer pt_out = null;
        SequenceFile.Writer g_out = null;

View Full Code Here


      // Get the old copy out of the way
      if (fs.exists(segmentDumpFile)) fs.delete(segmentDumpFile);


      final PrintStream printStream = new PrintStream(fs.create(segmentDumpFile));
      return new RecordWriter() {
        public synchronized void write(WritableComparable key, Writable value) throws IOException {
          ObjectWritable writable = (ObjectWritable) value;
          printStream.println((String) writable.get());
        }

View Full Code Here

      new Path(new Path(job.getOutputPath(), Content.DIR_NAME), name);


    final MapFile.Writer fetchOut =
      new MapFile.Writer(job, fs, fetch.toString(), Text.class, CrawlDatum.class);
    
    return new RecordWriter() {
        private MapFile.Writer contentOut;
        private RecordWriter parseOut;


        {
          if (Fetcher.isStoringContent(job)) {

View Full Code Here

        job.setOutputPath(output);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(CrawlDatum.class);
        // job.setOutputFormat(SequenceFileOutputFormat.class);
        // job.setOutputKeyComparatorClass(HashComparator.class);
        RecordWriter writer = new SequenceFileOutputFormat().getRecordWriter(null,job,"fetcher",new NoProgress());
        for (com.flaptor.hounder.crawler.pagedb.Page page : fetchlist) {
            Text key = new Text(page.getUrl());
            CrawlDatum value = new CrawlDatum(); // TODO: try taking this line outside of the loop
            writer.write(key,value);
        }
        writer.close(null);
        return segmentDir;
    }

View Full Code Here

          ObjectInspectorFactory.getReflectionObjectInspector(NestedRow.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
    SerDe serde = new OrcSerde();
    OutputFormat<?, ?> outFormat = new OrcOutputFormat();
    RecordWriter writer =
        outFormat.getRecordWriter(fs, conf, testFilePath.toString(),
            Reporter.NULL);
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(1,2,3), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(4,5,6), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(7,8,9), inspector));
    writer.close(Reporter.NULL);
    serde = new OrcSerde();
    properties.setProperty("columns", "z,r");
    properties.setProperty("columns.types", "int:struct<x:int,y:int>");
    serde.initialize(conf, properties);
    inspector = (StructObjectInspector) serde.getObjectInspector();

View Full Code Here

  // MultithreaderMapRunner.
  private synchronized RecordWriter getRecordWriter(String namedOutput,
                                                    String baseFileName,
                                                    final Reporter reporter)
    throws IOException {
    RecordWriter writer = recordWriters.get(baseFileName);
    if (writer == null) {
      if (countersEnabled && reporter == null) {
        throw new IllegalArgumentException(
          "Counters are enabled, Reporter cannot be NULL");
      }

View Full Code Here

      checkTokenName(multiName);
    }


    String baseFileName = (multi) ? namedOutput + "_" + multiName : namedOutput;


    final RecordWriter writer =
      getRecordWriter(namedOutput, baseFileName, reporter);


    return new AvroCollector() {
   
      @SuppressWarnings({"unchecked"})
      public void collect(Object key) throws IOException{
       AvroWrapper wrapper = new AvroWrapper(key);
       writer.write(wrapper, NullWritable.get());
      }
      
      public void collect(Object key,Object value) throws IOException
      {
        writer.write(key,value);
      }  
    
    };
  }

View Full Code Here

          ObjectInspectorFactory.getReflectionObjectInspector(StringRow.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
    SerDe serde = new OrcSerde();
    OutputFormat<?, ?> outFormat = new OrcOutputFormat();
    RecordWriter writer =
        outFormat.getRecordWriter(fs, job, testFilePath.getName(),
            Reporter.NULL);
    writer.write(NullWritable.get(),
        serde.serialize(new StringRow("a"), inspector));
    writer.close(Reporter.NULL);
    serde = new OrcSerde();
    properties.setProperty("columns", "col");
    properties.setProperty("columns.types", "string");
    serde.initialize(conf, properties);
    inspector = (StructObjectInspector) serde.getObjectInspector();

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.hadoop.mapred.RecordWriter

com.facebook.hive.orc.TestInputOutputFormat

com.flaptor.hounder.crawler.Nutch9Fetcher

org.apache.avro.mapred.AvroMultipleOutputs

org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat

org.apache.hadoop.mapred.lib.NullOutputFormat

org.apache.hive.hcatalog.mapreduce.DynamicPartitionFileRecordWriterContainer

org.apache.hive.hcatalog.mapreduce.FileRecordWriterContainer

org.apache.nutch.fetcher.FetcherOutputFormat

org.apache.nutch.segment.SegmentMerger$SegmentOutputFormat

org.apache.nutch.segment.SegmentReader$TextOutputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.