Examples of org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe

Package org.apache.hadoop.hive.serde2.binarysortable

Examples of org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe

org.apache.hadoop.hive.serde2.io.DoubleWritable
BinarySortableSerDe can be used to write data in a way that the data can be compared byte-by-byte with the same order. The data format: NULL: a single byte \0 NON-NULL Primitives: ALWAYS prepend a single byte \1, and then: Boolean: FALSE = \1, TRUE = \2 Byte: flip the sign-bit to make sure negative comes before positive Short: flip the sign-bit to make sure negative comes before positive Int: flip the sign-bit to make sure negative comes before positive Long: flip the sign-bit to make sure negative comes before positive Double: flip the sign-bit for positive double, and all bits for negative double values String: NULL-terminated UTF-8 string, with NULL escaped to \1 \1, and \1 escaped to \1 \2 NON-NULL Complex Types: ALWAYS prepend a single byte \1, and then: Struct: one field by one field. List: \1 followed by each element, and \0 to terminate Map: \1 followed by each key and then each value, and \0 to terminate This SerDe takes an additional parameter SERIALIZATION_SORT_ORDER which is a string containing only "+" and "-". The length of the string should equal to the number of fields in the top-level struct for serialization. "+" means the field should be sorted ascendingly, and "-" means descendingly. The sub fields in the same top-level field will have the same sort order.

      this.conf = conf;
      this.stripeSize = stripeSize;
      this.compress = CompressionKind.valueOf(compress);
      this.compressionSize = compressionSize;
      this.rowIndexStride = rowIndexStride;
      this.stats = new SerDeStats();
    }

View Full Code Here

   * 
   * @param typeInfo
   * @return
   */
  public static BinarySortableSerDe createBinarySerde(TypeInfo typeInfo){
    BinarySortableSerDe serde = new BinarySortableSerDe();
    
    StringBuffer nameSb = new StringBuffer();
    StringBuffer typeSb = new StringBuffer();


    StructTypeInfo sti = (StructTypeInfo) typeInfo;
    for (String name : sti.getAllStructFieldNames()) {
      nameSb.append(name);
      nameSb.append(',');
    }
    for (TypeInfo info : sti.getAllStructFieldTypeInfos()) {
      typeSb.append(info.toString());
      typeSb.append(',');
    }


    Properties tbl = new Properties();
    String names = nameSb.length() > 0 ? nameSb.substring(0,
        nameSb.length() - 1) : "";
    String types = typeSb.length() > 0 ? typeSb.substring(0,
        typeSb.length() - 1) : "";
    tbl.setProperty(serdeConstants.LIST_COLUMNS, names);
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, types);
    
    try {
      serde.initialize(null, tbl);
    } catch (SerDeException e) {
      throw new CrunchRuntimeException("Unable to initialize binary serde");
    }
    
    return serde;

View Full Code Here

  @Test
  public void testDeepCopy() {
    String typeStr = "struct<a:int,b:string,c:float>";
    TypeInfo info = TypeInfoUtils.getTypeInfoFromTypeString(typeStr);
    StructObjectInspector oi = (StructObjectInspector) OrcStruct.createObjectInspector(info);
    BinarySortableSerDe serde = OrcUtils.createBinarySerde(info);
    
    OrcStruct struct = OrcUtils.createOrcStruct(info,
        new IntWritable(1), new Text("Alice"), new FloatWritable(165.3f));
    OrcWritable writable = new OrcWritable();
    writable.set(struct);

View Full Code Here

  @Test
  public void testCompareTo() {
    String typeStr = "struct<a:int,b:string,c:float>";
    TypeInfo info = TypeInfoUtils.getTypeInfoFromTypeString(typeStr);
    StructObjectInspector oi = (StructObjectInspector) OrcStruct.createObjectInspector(info);
    BinarySortableSerDe serde = OrcUtils.createBinarySerde(info);
    
    OrcStruct struct1 = OrcUtils.createOrcStruct(info, new IntWritable(1), new Text("AAA"), new FloatWritable(3.2f));
    OrcStruct struct2 = OrcUtils.createOrcStruct(info, new IntWritable(1), new Text("AAB"), null);
    OrcStruct struct3 = OrcUtils.createOrcStruct(info, new IntWritable(2), new Text("AAA"), null);
    OrcStruct struct4 = OrcUtils.createOrcStruct(info, new IntWritable(2), new Text("AAA"), new FloatWritable(3.2f));

View Full Code Here

    return reader.loadIndeces(rowIndexEntries, updatedStartIndex);
  }


  @Override
  public Object next(Object previous) throws IOException {
    ByteWritable result = null;
    if (valuePresent) {
      if (previous == null) {
        result = new ByteWritable();
      } else {
        result = (ByteWritable) previous;
      }
      result.set(reader.next());
    }
    return result;
  }

View Full Code Here

    return o == null ? null : new OrcLazyByte((OrcLazyByte) o);
  }


  @Override
  public Object getPrimitiveJavaObject(Object o) {
    ByteWritable writable = getPrimitiveWritableObject(o);
    return writable == null ? null : Byte.valueOf(writable.get());
  }

View Full Code Here

  }


  public OrcLazyByte(OrcLazyByte copy) {
    super(copy);
    if (copy.previous != null) {
      previous = new ByteWritable(((ByteWritable)copy.previous).get());
    }
  }

View Full Code Here

      int nextCalls = 0;


      @Override
      public Object next(Object previous) throws IOException {
        if (nextCalls == 0) {
          return new ByteWritable((byte) 1);
        }


        throw new IOException("next should only be called once");
      }

View Full Code Here

      assertNull(expected.boolean1);
    } else {
      assertEquals(expected.boolean1.booleanValue(), boolean1.get());
    }


    ByteWritable byte1 = (ByteWritable) ((OrcLazyByte) row.getFieldValue(1)).materialize();
    if (byte1 == null) {
      assertNull(expected.byte1);
    } else {
      assertEquals(expected.byte1.byteValue(), byte1.get());
    }


    OrcLazyShort lazyShort1 = (OrcLazyShort) row.getFieldValue(2);
    ShortWritable short1 = (ShortWritable) lazyShort1.materialize();
    if (short1 == null) {

View Full Code Here

    return latestRead;
  }




  DoubleWritable createWritable(Object previous, double v) throws IOException {
    DoubleWritable result = null;
    if (previous == null) {
      result = new DoubleWritable();
    } else {
      result = (DoubleWritable) previous;
    }
    result.set(v);
    return result;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe

com.asakusafw.directio.hive.orc.OrcFileInput

com.asakusafw.directio.hive.serde.DataModelDriver

com.asakusafw.directio.hive.serde.DataModelDriverTest

com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest

com.facebook.hive.orc.lazy.LazyDoubleTreeReader

com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader

com.facebook.hive.orc.lazy.LazyShortDirectTreeReader

com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector

com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter

com.facebook.hive.orc.OrcStruct$OrcStructInspector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.