Examples of org.apache.hadoop.io.serializer.SerializationFactory

org.apache.hadoop.io.serializer.SerializationFactory

A factory for {@link Serialization}s.

    Class<? extends InputSplit> inputSplitClass = readClass(in);
    inputSplit = (InputSplit) ReflectionUtils.newInstance(inputSplitClass, conf);
    if (inputSplit instanceof Writable) {
      ((Writable) inputSplit).readFields(in);
    } else {
      SerializationFactory factory = new SerializationFactory(conf);
      Deserializer deserializer = factory.getDeserializer(inputSplitClass);
      deserializer.open((DataInputStream) in);
      inputSplit = (InputSplit) deserializer.deserialize(inputSplit);
      deserializer.close();
    }
  }

View Full Code Here

    bundle.write(out);
    Text.writeString(out, inputSplit.getClass().getName());
    if (inputSplit instanceof Writable) {
      ((Writable) inputSplit).write(out);
    } else {
      SerializationFactory factory = new SerializationFactory(conf);
      Serializer serializer = factory.getSerializer(inputSplit.getClass());
      serializer.open((OutputStream) out);
      serializer.serialize(inputSplit);
      serializer.close();
    }
  }

View Full Code Here

                           Progressable reporter)
      throws IOException {
      this.in = in;
      this.comparator = comparator;
      this.reporter = reporter;
      SerializationFactory serializationFactory = new SerializationFactory(conf);
      this.keyDeserializer = serializationFactory.getDeserializer(keyClass);
      this.keyDeserializer.open(keyIn);
      this.valDeserializer = serializationFactory.getDeserializer(valClass);
      this.valDeserializer.open(this.valueIn);
      readNextKey();
      key = nextKey;
      nextKey = null; // force new instance creation
      hasNext = more;

View Full Code Here

        targetOps = (ArrayList<OperatorKey>) readObject(is);
        String splitClassName = is.readUTF();
        try {
            Class splitClass = conf.getClassByName(splitClassName);
            wrappedSplit = (InputSplit)ReflectionUtils.newInstance(splitClass, conf);
            SerializationFactory sf = new SerializationFactory(conf);
            // The correct call sequence for Deserializer is, we shall open, then deserialize, but we shall not close
            Deserializer d = sf.getDeserializer(splitClass);
            d.open((InputStream) is);
            d.deserialize(wrappedSplit);
        } catch (ClassNotFoundException e) {
            throw new IOException(e);
        }

View Full Code Here

    public void write(DataOutput os) throws IOException {
        os.writeInt(splitIndex);
        os.writeInt(inputIndex);
        writeObject(targetOps, os);
        os.writeUTF(wrappedSplit.getClass().getName());
        SerializationFactory sf = new SerializationFactory(conf);
        Serializer s = 
            sf.getSerializer(wrappedSplit.getClass());
        // The correct call sequence for Serializer is, we shall open, then serialize, but we shall not close
        s.open((OutputStream) os);
        s.serialize(wrappedSplit);
        
    }

View Full Code Here

              ReflectionUtils.newInstance(jContext.getInputFormatClass(), jContext.getJobConf());
                    
          List<org.apache.hadoop.mapreduce.InputSplit> splits = input.getSplits(jContext);
          rawSplits = new RawSplit[splits.size()];
          DataOutputBuffer buffer = new DataOutputBuffer();
          SerializationFactory factory = new SerializationFactory(conf);
          Serializer serializer = 
            factory.getSerializer(splits.get(0).getClass());
          serializer.open(buffer);
          for (int i = 0; i < splits.size(); i++) {
            buffer.reset();
            serializer.serialize(splits.get(i));
            RawSplit rawSplit = new RawSplit();

View Full Code Here

      LOG.info("record buffer = " + softRecordLimit + "/" + kvoffsets.length);
      // k/v serialization
      comparator = job.getOutputKeyComparator();
      keyClass = (Class<K>)job.getMapOutputKeyClass();
      valClass = (Class<V>)job.getMapOutputValueClass();
      serializationFactory = new SerializationFactory(job);
      keySerializer = serializationFactory.getSerializer(keyClass);
      keySerializer.open(bb);
      valSerializer = serializationFactory.getSerializer(valClass);
      valSerializer.open(bb);
      // counters

View Full Code Here

                                                 array.length);
    try {
      if (array.length != 0) {
        DataOutputBuffer buffer = new DataOutputBuffer();
        RawSplit rawSplit = new RawSplit();
        SerializationFactory factory = new SerializationFactory(conf);
        Serializer<T> serializer = 
          factory.getSerializer((Class<T>) array[0].getClass());
        serializer.open(buffer);
        for(T split: array) {
          rawSplit.setClassName(split.getClass().getName());
          buffer.reset();
          serializer.serialize(split);

View Full Code Here

        ReflectionUtils.newInstance(taskContext.getInputFormatClass(), job);
    // rebuild the input split
    org.apache.hadoop.mapreduce.InputSplit split = null;
    DataInputBuffer splitBuffer = new DataInputBuffer();
    splitBuffer.reset(rawSplit.getBytes(), 0, rawSplit.getLength());
    SerializationFactory factory = new SerializationFactory(job);
    Deserializer<? extends org.apache.hadoop.mapreduce.InputSplit>
      deserializer = 
        (Deserializer<? extends org.apache.hadoop.mapreduce.InputSplit>) 
        factory.getDeserializer(job.getClassByName(splitClass));
    deserializer.open(splitBuffer);
    split = deserializer.deserialize(null);


    org.apache.hadoop.mapreduce.RecordReader<INKEY,INVALUE> input =
      new NewTrackingRecordReader<INKEY,INVALUE>

View Full Code Here

                           Progressable reporter)
      throws IOException {
      this.in = in;
      this.comparator = comparator;
      this.reporter = reporter;
      SerializationFactory serializationFactory = new SerializationFactory(conf);
      this.keyDeserializer = serializationFactory.getDeserializer(keyClass);
      this.keyDeserializer.open(keyIn);
      this.valDeserializer = serializationFactory.getDeserializer(valClass);
      this.valDeserializer.open(this.valueIn);
      readNextKey();
      key = nextKey;
      nextKey = null; // force new instance creation
      hasNext = more;

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.io.serializer.SerializationFactory

cascading.tuple.hadoop.TupleSerialization

com.datasalt.pangool.serialization.HadoopSerialization

com.datasalt.pangool.tuplemr.mapred.lib.input.TaggedInputSplit

com.manning.hip.ch13.TaskSplitReader

com.tellapart.cascading.hbase.HBaseSerialization

com.twitter.elephantbird.util.SplitUtil

com.twitter.maple.tap.TupleMemoryInputFormat

org.apache.accumulo.core.file.map.MySequenceFile$Reader

org.apache.accumulo.core.file.map.MySequenceFile$Writer

org.apache.crunch.impl.mr.run.CrunchInputSplit

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.