Examples of org.apache.hadoop.mapred.RecordWriter

org.apache.hadoop.mapred.RecordWriter
RecordWriter writes the output <key, value> pairs to an output file.
RecordWriter implementations write the job outputs to the {@link FileSystem}. @see OutputFormat

          ObjectInspectorFactory.getReflectionObjectInspector(NestedRow.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
    SerDe serde = new OrcSerde();
    OutputFormat<?, ?> outFormat = new OrcOutputFormat();
    RecordWriter writer =
        outFormat.getRecordWriter(fs, conf, testFilePath.toString(),
            Reporter.NULL);
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(1,2,3), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(4,5,6), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(7,8,9), inspector));
    writer.close(Reporter.NULL);
    serde = new OrcSerde();
    properties.setProperty("columns", "z,r");
    properties.setProperty("columns.types", "int:struct<x:int,y:int>");
    serde.initialize(conf, properties);
    inspector = (StructObjectInspector) serde.getObjectInspector();

View Full Code Here

          ObjectInspectorFactory.getReflectionObjectInspector(NestedRow.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
    SerDe serde = new OrcSerde();
    OutputFormat<?, ?> outFormat = new OrcOutputFormat();
    RecordWriter writer =
        outFormat.getRecordWriter(fs, conf, testFilePath.toString(),
            Reporter.NULL);
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(1,2,3), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(4,5,6), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(7,8,9), inspector));
    writer.close(Reporter.NULL);
    serde = new OrcSerde();
    properties.setProperty("columns", "z,r");
    properties.setProperty("columns.types", "int:struct<x:int,y:int>");
    serde.initialize(conf, properties);
    inspector = (StructObjectInspector) serde.getObjectInspector();

View Full Code Here

  // MultithreaderMapRunner.
  private synchronized RecordWriter getRecordWriter(String namedOutput,
                                                    String baseFileName,
                                                    final Reporter reporter)
    throws IOException {
    RecordWriter writer = recordWriters.get(baseFileName);
    if (writer == null) {
      if (countersEnabled && reporter == null) {
        throw new IllegalArgumentException(
          "Counters are enabled, Reporter cannot be NULL");
      }

View Full Code Here

      checkTokenName(multiName);
    }


    String baseFileName = (multi) ? namedOutput + "_" + multiName : namedOutput;


    final RecordWriter writer =
      getRecordWriter(namedOutput, baseFileName, reporter);


    return new AvroCollector() {
   
      @SuppressWarnings({"unchecked"})
      public void collect(Object key) throws IOException{
       AvroWrapper wrapper = new AvroWrapper(key);
       writer.write(wrapper, NullWritable.get());
      }
      
      public void collect(Object key,Object value) throws IOException
      {
        writer.write(key,value);
      }  
    
    };
  }

View Full Code Here

          ObjectInspectorFactory.getReflectionObjectInspector(NestedRow.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
    SerDe serde = new OrcSerde();
    OutputFormat<?, ?> outFormat = new OrcOutputFormat();
    RecordWriter writer =
        outFormat.getRecordWriter(fs, conf, testFilePath.toString(),
            Reporter.NULL);
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(1,2,3), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(4,5,6), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(7,8,9), inspector));
    writer.close(Reporter.NULL);
    serde = new OrcSerde();
    properties.setProperty("columns", "z,r");
    properties.setProperty("columns.types", "int:struct<x:int,y:int>");
    serde.initialize(conf, properties);
    inspector = (StructObjectInspector) serde.getObjectInspector();

View Full Code Here

      new Path(new Path(job.getOutputPath(), Content.DIR_NAME), name);


    final MapFile.Writer fetchOut =
      new MapFile.Writer(fs, fetch.toString(), UTF8.class, CrawlDatum.class);
    
    return new RecordWriter() {
        private MapFile.Writer contentOut;
        private RecordWriter parseOut;


        {
          if (Fetcher.isStoringContent(job)) {

View Full Code Here

          ObjectInspectorFactory.getReflectionObjectInspector(NestedRow.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
    SerDe serde = new OrcSerde();
    OutputFormat<?, ?> outFormat = new OrcOutputFormat();
    RecordWriter writer =
        outFormat.getRecordWriter(fs, conf, testFilePath.toString(),
            Reporter.NULL);
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(1,2,3), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(4,5,6), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(7,8,9), inspector));
    writer.close(Reporter.NULL);
    serde = new OrcSerde();
    properties.setProperty("columns", "z,r");
    properties.setProperty("columns.types", "int:struct<x:int,y:int>");
    serde.initialize(conf, properties);
    inspector = (StructObjectInspector) serde.getObjectInspector();

View Full Code Here


  @Override
  public void write(WritableComparable<?> key, HCatRecord value) throws IOException,
      InterruptedException {
    LocalFileWriter localFileWriter = getLocalFileWriter(value);
    RecordWriter localWriter = localFileWriter.getLocalWriter();
    ObjectInspector localObjectInspector = localFileWriter.getLocalObjectInspector();
    SerDe localSerDe = localFileWriter.getLocalSerDe();
    OutputJobInfo localJobInfo = localFileWriter.getLocalJobInfo();


    for (Integer colToDel : partColsToDel) {
      value.remove(colToDel);
    }


    // The key given by user is ignored
    try {
      localWriter.write(NullWritable.get(),
          localSerDe.serialize(value.getAll(), localObjectInspector));
    } catch (SerDeException e) {
      throw new IOException("Failed to serialize object", e);
    }
  }

View Full Code Here


      Path parentDir = new Path(currTaskContext.getConfiguration().get("mapred.work.output.dir"));
      Path childPath =
          new Path(parentDir, FileOutputFormat.getUniqueFile(currTaskContext, "part", ""));


      RecordWriter baseRecordWriter =
          baseOF.getRecordWriter(parentDir.getFileSystem(currTaskContext.getConfiguration()),
              currTaskContext.getJobConf(), childPath.toString(),
              InternalUtil.createReporter(currTaskContext));


      baseDynamicWriters.put(dynKey, baseRecordWriter);

View Full Code Here

          ObjectInspectorFactory.getReflectionObjectInspector(NestedRow.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }
    SerDe serde = new OrcSerde();
    OutputFormat<?, ?> outFormat = new OrcOutputFormat();
    RecordWriter writer =
        outFormat.getRecordWriter(fs, conf, testFilePath.toString(),
            Reporter.NULL);
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(1,2,3), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(4,5,6), inspector));
    writer.write(NullWritable.get(),
        serde.serialize(new NestedRow(7,8,9), inspector));
    writer.close(Reporter.NULL);
    serde = new OrcSerde();
    properties.setProperty("columns", "z,r");
    properties.setProperty("columns.types", "int:struct<x:int,y:int>");
    SerDeUtils.initializeSerDe(serde, conf, properties, null);
    inspector = (StructObjectInspector) serde.getObjectInspector();

View Full Code Here

0 1 2 3

TOP

Related Classes of org.apache.hadoop.mapred.RecordWriter

com.facebook.hive.orc.TestInputOutputFormat

com.flaptor.hounder.crawler.Nutch9Fetcher

org.apache.avro.mapred.AvroMultipleOutputs

org.apache.hadoop.hive.ql.io.orc.TestInputOutputFormat

org.apache.hadoop.mapred.lib.NullOutputFormat

org.apache.hive.hcatalog.mapreduce.DynamicPartitionFileRecordWriterContainer

org.apache.hive.hcatalog.mapreduce.FileRecordWriterContainer

org.apache.nutch.fetcher.FetcherOutputFormat

org.apache.nutch.segment.SegmentMerger$SegmentOutputFormat

org.apache.nutch.segment.SegmentReader$TextOutputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.