Package org.apache.hadoop.io.serializer

Examples of org.apache.hadoop.io.serializer.SerializationFactory

81
82
83
84
85
86
87
88
89
90
91
92
    } catch (ClassNotFoundException ce) {
      IOException wrap = new IOException("Split class " + className + " not found");
      wrap.initCause(ce);
      throw wrap;
    }
    SerializationFactory factory = new SerializationFactory(jobConf);
    Deserializer<org.apache.hadoop.mapreduce.InputSplit> deserializer = (Deserializer<org.apache.hadoop.mapreduce.InputSplit>) factory
        .getDeserializer(cls);
    deserializer.open(inFile);
    org.apache.hadoop.mapreduce.InputSplit split = deserializer.deserialize(null);
    long pos = inFile.getPos();
    if (splitBytesCounter != null) {
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
125
    } catch (ClassNotFoundException ce) {
      IOException wrap = new IOException("Split class " + className + " not found");
      wrap.initCause(ce);
      throw wrap;
    }
    SerializationFactory factory = new SerializationFactory(jobConf);
    Deserializer<org.apache.hadoop.mapred.InputSplit> deserializer = (Deserializer<org.apache.hadoop.mapred.InputSplit>) factory
        .getDeserializer(cls);
    deserializer.open(inFile);
    org.apache.hadoop.mapred.InputSplit split = deserializer.deserialize(null);
    long pos = inFile.getPos();
    if (splitBytesCounter != null) {
View Full Code Here
130
131
132
133
134
135
136
137
  }
 
  @Private
  public static InputSplit getOldSplitDetailsFromEvent(MRSplitProto splitProto, Configuration conf)
      throws IOException {
    SerializationFactory serializationFactory = new SerializationFactory(conf);
    return MRInputHelpers.createOldFormatSplitFromUserPayload(splitProto, serializationFactory);
  }
View Full Code Here
517
518
519
520
521
522
523
524
525
526
527
528
529
530
    IntWritable readValue = new IntWritable();
    DataInputBuffer keyIn = new DataInputBuffer();
    DataInputBuffer valIn = new DataInputBuffer();
    Deserializer<Text> keyDeserializer;
    Deserializer<IntWritable> valDeserializer;
    SerializationFactory serializationFactory = new SerializationFactory(
        defaultConf);
    keyDeserializer = serializationFactory.getDeserializer(Text.class);
    valDeserializer = serializationFactory.getDeserializer(IntWritable.class);
    keyDeserializer.open(keyIn);
    valDeserializer.open(valIn);

    int numRecordsRead = 0;
View Full Code Here
171
172
173
174
175
176
177
178
179
180
181
182
183
      writeHeader(outputStream);
      this.keyClass = keyClass;
      this.valueClass = valueClass;

      if (keyClass != null) {
        SerializationFactory serializationFactory =
          new SerializationFactory(conf);
        this.keySerializer = serializationFactory.getSerializer(keyClass);
        this.keySerializer.open(buffer);
        this.valueSerializer = serializationFactory.getSerializer(valueClass);
        this.valueSerializer.open(buffer);
      }
    }
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
    this.valClass = ConfigUtils.getIntermediateInputValueClass(conf);

    this.keyIn = new DataInputBuffer();
    this.valIn = new DataInputBuffer();

    SerializationFactory serializationFactory = new SerializationFactory(conf);

    this.keyDeserializer = serializationFactory.getDeserializer(keyClass);
    this.keyDeserializer.open(keyIn);
    this.valDeserializer = serializationFactory.getDeserializer(valClass);
    this.valDeserializer.open(valIn);
  }
View Full Code Here
73
74
75
76
77
78
79
80
81
82
83
84
    throws IOException {
    this.in = in;
    this.comparator = comparator;
    this.inputKeyCounter = inputKeyCounter;
    this.inputValueCounter = inputValueCounter;
    SerializationFactory serializationFactory = new SerializationFactory(conf);
    this.keyDeserializer = serializationFactory.getDeserializer(keyClass);
    this.keyDeserializer.open(keyIn);
    this.valDeserializer = serializationFactory.getDeserializer(valClass);
    this.valDeserializer.open(this.valueIn);
    LOG.info("keyDeserializer=" + keyDeserializer + "; valueDeserializer=" + valDeserializer);
  }
View Full Code Here
159
160
161
162
163
164
165
166
167
168
169
    comparator = ConfigUtils.getIntermediateOutputKeyComparator(this.conf);

    // k/v serialization
    keyClass = ConfigUtils.getIntermediateOutputKeyClass(this.conf);
    valClass = ConfigUtils.getIntermediateOutputValueClass(this.conf);
    serializationFactory = new SerializationFactory(this.conf);
    keySerializer = serializationFactory.getSerializer(keyClass);
    valSerializer = serializationFactory.getSerializer(valClass);
    LOG.info("keySerializer=" + keySerializer + "; valueSerializer=" + valSerializer
        + "; comparator=" + (RawComparator) ConfigUtils.getIntermediateOutputKeyComparator(conf)
        + "; conf=" + conf.get(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY));
View Full Code Here
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
      this.keyClass = keyClass;
      this.valClass = valClass;
      this.compress = compress;
      this.codec = codec;
      this.metadata = metadata;
      SerializationFactory serializationFactory = new SerializationFactory(conf);
      this.keySerializer = serializationFactory.getSerializer(keyClass);
      this.keySerializer.open(buffer);
      this.uncompressedValSerializer = serializationFactory.getSerializer(valClass);
      this.uncompressedValSerializer.open(buffer);
      if (this.codec != null) {
        ReflectionUtils.setConf(this.codec, this.conf);
        this.compressor = CodecPool.getCompressor(this.codec);
        this.deflateFilter = this.codec.createOutputStream(buffer, compressor);
        this.deflateOut =
          new DataOutputStream(new BufferedOutputStream(deflateFilter));
        this.compressedValSerializer = serializationFactory.getSerializer(valClass);
        this.compressedValSerializer.open(deflateOut);
      }
    }
View Full Code Here
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
          valLenInFilter = codec.createInputStream(valLenBuffer,
                                                   valLenDecompressor);
          valLenIn = new DataInputStream(valLenInFilter);
        }
       
        SerializationFactory serializationFactory =
          new SerializationFactory(conf);
        this.keyDeserializer =
          getDeserializer(serializationFactory, getKeyClass());
        if (!blockCompressed) {
          this.keyDeserializer.open(valBuffer);
        } else {
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.serializer.SerializationFactory

  • cascading.tuple.hadoop.TupleSerialization
  • com.datasalt.pangool.serialization.HadoopSerialization
  • com.datasalt.pangool.tuplemr.mapred.lib.input.TaggedInputSplit
  • com.manning.hip.ch13.TaskSplitReader
  • com.tellapart.cascading.hbase.HBaseSerialization
  • com.twitter.elephantbird.util.SplitUtil
  • com.twitter.maple.tap.TupleMemoryInputFormat
  • org.apache.accumulo.core.file.map.MySequenceFile$Reader
  • org.apache.accumulo.core.file.map.MySequenceFile$Writer
  • org.apache.crunch.impl.mr.run.CrunchInputSplit
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.