Examples of org.apache.hadoop.io.serializer.Serializer

org.apache.hadoop.io.serializer.Serializer

Provides a facility for serializing objects of type to an {@link OutputStream}.

Serializers are stateful, but must not buffer the output since other producers may write to the output between calls to {@link #serialize(Object)}.
@param < T>

  public List<TezMerger.Segment> createInMemStreams() throws IOException {
    int numberOfStreams = Math.max(2, rnd.nextInt(5));
    LOG.info("No of streams : " + numberOfStreams);


    SerializationFactory serializationFactory = new SerializationFactory(conf);
    Serializer keySerializer = serializationFactory.getSerializer(keyClass);
    Serializer valueSerializer = serializationFactory.getSerializer(valClass);


    LocalDirAllocator localDirAllocator =
        new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS);
    InputContext context = createTezInputContext();
    MergeManager mergeManager = new MergeManager(conf, fs, localDirAllocator,
        context, null, null, null, null, null, 1024 * 1024 * 10, null, false, -1);


    DataOutputBuffer keyBuf = new DataOutputBuffer();
    DataOutputBuffer valBuf = new DataOutputBuffer();
    DataInputBuffer keyIn = new DataInputBuffer();
    DataInputBuffer valIn = new DataInputBuffer();
    keySerializer.open(keyBuf);
    valueSerializer.open(valBuf);


    List<TezMerger.Segment> segments = new LinkedList<TezMerger.Segment>();
    for (int i = 0; i < numberOfStreams; i++) {
      BoundedByteArrayOutputStream bout = new BoundedByteArrayOutputStream(1024 * 1024);
      InMemoryWriter writer =
          new InMemoryWriter(bout);
      Map<Writable, Writable> data = createData();
      //write data
      for (Map.Entry<Writable, Writable> entry : data.entrySet()) {
        keySerializer.serialize(entry.getKey());
        valueSerializer.serialize(entry.getValue());
        keyIn.reset(keyBuf.getData(), 0, keyBuf.getLength());
        valIn.reset(valBuf.getData(), 0, valBuf.getLength());
        writer.append(keyIn, valIn);
        keyBuf.reset();
        valBuf.reset();

View Full Code Here

    Text.writeString(out, inputSplit.getClass().getName());
    if (inputSplit instanceof Writable) {
      ((Writable) inputSplit).write(out);
    } else {
      SerializationFactory factory = new SerializationFactory(conf);
      Serializer serializer = factory.getSerializer(inputSplit.getClass());
      serializer.open((OutputStream) out);
      serializer.serialize(inputSplit);
      serializer.close();
    }
  }

View Full Code Here

        os.writeInt(inputIndex);
        writeObject(targetOps, os);
        os.writeInt(wrappedSplits.length);
        os.writeUTF(wrappedSplits[0].getClass().getName());
        SerializationFactory sf = new SerializationFactory(conf);
        Serializer s = 
            sf.getSerializer(wrappedSplits[0].getClass());
         
        //Checks if Serializer is NULL or not before calling open() method on it.         
        if (s == null) {
              throw new IllegalArgumentException("Could not find Serializer for class "+wrappedSplits[0].getClass()+". InputSplits must implement Writable.");
        }        
        s.open((OutputStream) os);
        for (int i = 0; i < wrappedSplits.length; i++)
        {
            // The correct call sequence for Serializer is, we shall open, then serialize, but we shall not close
            s.serialize(wrappedSplits[i]);
        }
        
    }

View Full Code Here

  public void write(DataOutput out) throws IOException {
    out.writeInt(nodeIndex);
    bundle.write(out);
    Text.writeString(out, inputSplit.getClass().getName());
    SerializationFactory factory = new SerializationFactory(conf);
    Serializer serializer = factory.getSerializer(inputSplit.getClass());
    serializer.open((DataOutputStream) out);
    serializer.serialize(inputSplit);
  }

View Full Code Here

  public void write(DataOutput out) throws IOException {
    Text.writeString(out, inputSplitClass.getName());
    Text.writeString(out, inputFormatClass.getName());
    Text.writeString(out, mapperClass.getName());
    SerializationFactory factory = new SerializationFactory(conf);
    Serializer serializer = 
          factory.getSerializer(inputSplitClass);
    serializer.open((DataOutputStream)out);
    serializer.serialize(inputSplit);
  }

View Full Code Here

        os.writeInt(inputIndex);
        writeObject(targetOps, os);
        os.writeInt(wrappedSplits.length);
        os.writeUTF(wrappedSplits[0].getClass().getName());
        SerializationFactory sf = new SerializationFactory(conf);
        Serializer s = 
            sf.getSerializer(wrappedSplits[0].getClass());
        s.open((OutputStream) os);
        for (int i = 0; i < wrappedSplits.length; i++)
        {
            // The correct call sequence for Serializer is, we shall open, then serialize, but we shall not close
            s.serialize(wrappedSplits[i]);
        }
        
    }

View Full Code Here

      //
      // Write some data out to a flat file
      //
      FileInputFormat.setInputPaths(job, dir);
      ds = fs.create(file);
      Serializer serializer = new JavaSerialization().getSerializer(null);


      // construct some data and write it
      serializer.open(ds);
      for (int i = 0; i < 10; i++) {
        serializer.serialize(new JavaTestObjFlatFileInputFormat("Hello World! "
            + String.valueOf(i), i));
      }
      serializer.close();


      //
      // Construct the reader
      //
      FileInputFormat<Void, FlatFileInputFormat.RowContainer<Serializable>> format =

View Full Code Here

      //
      // Write some data out to a flat file
      //
      FileInputFormat.setInputPaths(job, dir);
      ds = fs.create(file);
      Serializer serializer = new WritableSerialization()
          .getSerializer(Writable.class);


      // construct some data and write it
      serializer.open(ds);
      for (int i = 0; i < 10; i++) {
        serializer.serialize(new RecordTestObj("Hello World! "
            + String.valueOf(i), i));
      }
      serializer.close();


      //
      // Construct the reader
      //
      FileInputFormat<Void, FlatFileInputFormat.RowContainer<Writable>> format =

View Full Code Here

      return null;
    }


    Class klazz = obj.getClass();
    Object out = null;
    Serializer s = serializationFactory.getSerializer(klazz);
    Deserializer ds = serializationFactory.getDeserializer(klazz);


    try {
      s.open(outBuffer);
      ds.open(inBuffer);


      outBuffer.reset();
      s.serialize(obj);


      byte [] data = outBuffer.getData();
      int len = outBuffer.getLength();
      inBuffer.reset(data, len);


      out = ds.deserialize(out);


      return out;
    } finally {
      try {
        s.close();
      } catch (IOException ioe) {
        // ignore this; we're closing.
      }


      try {

View Full Code Here

      //
      // Write some data out to a flat file
      //
      FileInputFormat.setInputPaths(job, dir);
      ds = fs.create(file);
      Serializer serializer = new JavaSerialization().getSerializer(null);


      // construct some data and write it
      serializer.open(ds);
      for (int i = 0; i < 10; i++) {
        serializer.serialize(new JavaTestObjFlatFileInputFormat("Hello World! "
            + String.valueOf(i), i));
      }
      serializer.close();


      //
      // Construct the reader
      //
      FileInputFormat<Void, FlatFileInputFormat.RowContainer<Serializable>> format =

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.apache.hadoop.io.serializer.Serializer

cascading.tuple.hadoop.TupleSerialization

cascading.tuple.hadoop.TupleSerialization$SerializationElementWriter

com.datasalt.pangool.serialization.HadoopSerialization

com.datasalt.pangool.tuplemr.mapred.lib.input.TaggedInputSplit

com.datasalt.pangool.tuplemr.mapred.SortComparator

com.datasalt.pangool.utils.TupleToAvroRecordConverter

com.tellapart.cascading.hbase.HBaseSerialization

com.twitter.elephantbird.util.SplitUtil

org.apache.crunch.impl.mr.run.CrunchInputSplit

org.apache.hadoop.hive.ql.io.TestFlatFileInputFormat

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.