Examples of org.apache.hadoop.io.serializer.Serializer.open()

Class org.apache.hadoop.io.serializer.Serializer

Examples of org.apache.hadoop.io.serializer.Serializer.open()

org.apache.hadoop.io.serializer.Serializer.open()

Prepare the serializer for writing.

    DataOutputBuffer keyBuf = new DataOutputBuffer();
    DataOutputBuffer valBuf = new DataOutputBuffer();
    DataInputBuffer keyIn = new DataInputBuffer();
    DataInputBuffer valIn = new DataInputBuffer();
    keySerializer.open(keyBuf);
    valueSerializer.open(valBuf);


    List<TezMerger.Segment> segments = new LinkedList<TezMerger.Segment>();
    for (int i = 0; i < numberOfStreams; i++) {
      BoundedByteArrayOutputStream bout = new BoundedByteArrayOutputStream(1024 * 1024);
      InMemoryWriter writer =

View Full Code Here

    if (inputSplit instanceof Writable) {
      ((Writable) inputSplit).write(out);
    } else {
      SerializationFactory factory = new SerializationFactory(conf);
      Serializer serializer = factory.getSerializer(inputSplit.getClass());
      serializer.open((OutputStream) out);
      serializer.serialize(inputSplit);
      serializer.close();
    }
  }

View Full Code Here

         
        //Checks if Serializer is NULL or not before calling open() method on it.         
        if (s == null) {
              throw new IllegalArgumentException("Could not find Serializer for class "+wrappedSplits[0].getClass()+". InputSplits must implement Writable.");
        }        
        s.open((OutputStream) os);
        for (int i = 0; i < wrappedSplits.length; i++)
        {
            // The correct call sequence for Serializer is, we shall open, then serialize, but we shall not close
            s.serialize(wrappedSplits[i]);
        }

View Full Code Here

    out.writeInt(nodeIndex);
    bundle.write(out);
    Text.writeString(out, inputSplit.getClass().getName());
    SerializationFactory factory = new SerializationFactory(conf);
    Serializer serializer = factory.getSerializer(inputSplit.getClass());
    serializer.open((DataOutputStream) out);
    serializer.serialize(inputSplit);
  }


  private Class readClass(DataInput in) throws IOException {
    String className = Text.readString(in);

View Full Code Here

    Text.writeString(out, inputFormatClass.getName());
    Text.writeString(out, mapperClass.getName());
    SerializationFactory factory = new SerializationFactory(conf);
    Serializer serializer = 
          factory.getSerializer(inputSplitClass);
    serializer.open((DataOutputStream)out);
    serializer.serialize(inputSplit);
  }


  public Configuration getConf() {
    return conf;

View Full Code Here

        os.writeInt(wrappedSplits.length);
        os.writeUTF(wrappedSplits[0].getClass().getName());
        SerializationFactory sf = new SerializationFactory(conf);
        Serializer s = 
            sf.getSerializer(wrappedSplits[0].getClass());
        s.open((OutputStream) os);
        for (int i = 0; i < wrappedSplits.length; i++)
        {
            // The correct call sequence for Serializer is, we shall open, then serialize, but we shall not close
            s.serialize(wrappedSplits[i]);
        }

View Full Code Here

      FileInputFormat.setInputPaths(job, dir);
      ds = fs.create(file);
      Serializer serializer = new JavaSerialization().getSerializer(null);


      // construct some data and write it
      serializer.open(ds);
      for (int i = 0; i < 10; i++) {
        serializer.serialize(new JavaTestObjFlatFileInputFormat("Hello World! "
            + String.valueOf(i), i));
      }
      serializer.close();

View Full Code Here

      ds = fs.create(file);
      Serializer serializer = new WritableSerialization()
          .getSerializer(Writable.class);


      // construct some data and write it
      serializer.open(ds);
      for (int i = 0; i < 10; i++) {
        serializer.serialize(new RecordTestObj("Hello World! "
            + String.valueOf(i), i));
      }
      serializer.close();

View Full Code Here

    Object out = null;
    Serializer s = serializationFactory.getSerializer(klazz);
    Deserializer ds = serializationFactory.getDeserializer(klazz);


    try {
      s.open(outBuffer);
      ds.open(inBuffer);


      outBuffer.reset();
      s.serialize(obj);

View Full Code Here

      FileInputFormat.setInputPaths(job, dir);
      ds = fs.create(file);
      Serializer serializer = new JavaSerialization().getSerializer(null);


      // construct some data and write it
      serializer.open(ds);
      for (int i = 0; i < 10; i++) {
        serializer.serialize(new JavaTestObjFlatFileInputFormat("Hello World! "
            + String.valueOf(i), i));
      }
      serializer.close();

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.