Examples of ByteArrayRef


Examples of org.apache.hadoop.hive.serde2.lazy.ByteArrayRef

            nulls[column] = false;
            if (hiveTypes[column] == HiveType.MAP || hiveTypes[column] == HiveType.LIST || hiveTypes[column] == HiveType.STRUCT) {
                // temporarily special case MAP, LIST, and STRUCT types as strings
                // TODO: create a real parser for these complex types when we implement data types
                LazyBinaryObject<? extends ObjectInspector> lazyObject = LazyBinaryFactory.createLazyBinaryObject(fieldInspectors[column]);
                ByteArrayRef byteArrayRef = new ByteArrayRef();
                byteArrayRef.setData(bytes);
                lazyObject.init(byteArrayRef, start, length);
                slices[column] = Slices.wrappedBuffer(SerDeUtils.getJsonBytes(sessionTimeZone, lazyObject.getObject(), fieldInspectors[column]));
            }
            else {
                // TODO: zero length BINARY is not supported. See https://issues.apache.org/jira/browse/HIVE-2483
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.ByteArrayRef

      // need to keep HCatStorer lean and go fast.
      Type type = hcatFS.getType();
      switch(type){

      case BINARY:
        ByteArrayRef ba = new ByteArrayRef();
        byte[] bytes = (null == pigObj) ? new byte[0] : ((DataByteArray)pigObj).get();
        ba.setData(bytes);
        return ba;

      case STRUCT:
        if (pigObj == null) {
          return null;
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.ByteArrayRef

    LazyObject<?> [] fields = getFields();
    boolean [] fieldsInited = getFieldInited();

    if (!fieldsInited[fieldID]) {
      fieldsInited[fieldID] = true;
      ByteArrayRef ref = null;
      ColumnMapping colMap = columnsMapping.get(fieldID);

      if (colMap.hbaseRowKey) {
        ref = new ByteArrayRef();
        ref.setData(result.getRow());
      } else {
        if (colMap.qualifierName == null) {
          // it is a column family
          // primitive type for Map<Key, Value> can be stored in binary format
          ((LazyHBaseCellMap) fields[fieldID]).init(
              result, colMap.familyNameBytes, colMap.binaryStorage);
        } else {
          // it is a column i.e. a column-family with column-qualifier
          byte [] res = result.getValue(colMap.familyNameBytes, colMap.qualifierNameBytes);

          if (res == null) {
            return null;
          } else {
            ref = new ByteArrayRef();
            ref.setData(res);
          }
        }
      }

      if (ref != null) {
        fields[fieldID].init(ref, 0, ref.getData().length);
      }
    }

    return fields[fieldID].getObject();
  }
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.ByteArrayRef

        LazyPrimitive<? extends ObjectInspector, ? extends Writable> key =
          LazyFactory.createLazyPrimitiveClass(
              (PrimitiveObjectInspector) lazyMoi.getMapKeyObjectInspector(),
              binaryStorage.get(0));

        ByteArrayRef keyRef = new ByteArrayRef();
        keyRef.setData(e.getKey());
        key.init(keyRef, 0, keyRef.getData().length);

        // Value
        LazyObject<?> value =
          LazyFactory.createLazyObject(lazyMoi.getMapValueObjectInspector(),
              binaryStorage.get(1));

        ByteArrayRef valueRef = new ByteArrayRef();
        valueRef.setData(e.getValue());
        value.init(valueRef, 0, valueRef.getData().length);

        // Put the key/value into the map
        cachedMap.put(key.getObject(), value.getObject());
      }
    }
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.ByteArrayRef

   * @throws Throwable
   */
  public void testLazyBinaryObjectInspector() throws Throwable {

    //create input ByteArrayRef
    ByteArrayRef inpBARef = new ByteArrayRef();
    inpBARef.setData(inpBArray);

    AbstractPrimitiveLazyObjectInspector<?> binInspector = LazyPrimitiveObjectInspectorFactory
    .getLazyObjectInspector(TypeInfoFactory.binaryTypeInfo, false, (byte)0);

    //create LazyBinary initialed with inputBA
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.ByteArrayRef

    if (c.isInstance(new Timestamp(0))){
      return javaObjectOverHead + javaSizePrimitiveType;
    }

    if (c.isInstance(new String()) || c.isInstance(new ByteArrayRef())) {
      if (aggrPositions[pos] == null) {
        aggrPositions[pos] = new ArrayList<Field>();
      }
      aggrPositions[pos].add(f);
      return javaObjectOverHead;
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.ByteArrayRef

    LazyObject<?> [] fields = getFields();
    boolean [] fieldsInited = getFieldInited();

    if (!fieldsInited[fieldID]) {
      ByteArrayRef ref = null;
      ColumnMapping colMap = columnsMapping.get(fieldID);

      if (colMap.hbaseRowKey) {
        ref = new ByteArrayRef();
        ref.setData(result.getRow());
      } else {
        if (colMap.qualifierName == null) {
          // it is a column family
          // primitive type for Map<Key, Value> can be stored in binary format. Pass in the
          // qualifier prefix to cherry pick the qualifiers that match the prefix instead of picking
          // up everything
          ((LazyHBaseCellMap) fields[fieldID]).init(
              result, colMap.familyNameBytes, colMap.binaryStorage, colMap.qualifierPrefixBytes);
        } else {
          // it is a column i.e. a column-family with column-qualifier
          byte [] res = result.getValue(colMap.familyNameBytes, colMap.qualifierNameBytes);

          if (res == null) {
            return null;
          } else {
            ref = new ByteArrayRef();
            ref.setData(res);
          }
        }
      }

      if (ref != null) {
        fields[fieldID].init(ref, 0, ref.getData().length);

        // if it was a row key and we have been provided a custom composite key class, initialize it
        // with the bytes for the row key
        if (colMap.hbaseRowKey && compositeKeyObj != null) {
          ((LazyStruct) compositeKeyObj).init(ref, 0, ref.getData().length);
        }
      }
    }

    // Has to be set last because of HIVE-3179: NULL fields would not work otherwise
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.ByteArrayRef

        LazyPrimitive<? extends ObjectInspector, ? extends Writable> key =
          LazyFactory.createLazyPrimitiveClass(
              (PrimitiveObjectInspector) lazyMoi.getMapKeyObjectInspector(),
              binaryStorage.get(0));

        ByteArrayRef keyRef = new ByteArrayRef();
        keyRef.setData(e.getKey());
        key.init(keyRef, 0, keyRef.getData().length);

        // Value
        LazyObject<?> value =
          LazyFactory.createLazyObject(lazyMoi.getMapValueObjectInspector(),
              binaryStorage.get(1));

        ByteArrayRef valueRef = new ByteArrayRef();
        valueRef.setData(e.getValue());
        value.init(valueRef, 0, valueRef.getData().length);

        // Put the key/value into the map
        cachedMap.put(key.getObject(), value.getObject());
      }
    }
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.ByteArrayRef

    ObjectInspector fieldOI = oi.getAllStructFieldRefs().get(fieldID).getFieldObjectInspector();

    LazyObject<? extends ObjectInspector> lazyObject = LazyFactory
        .createLazyObject(fieldOI);

    ByteArrayRef ref = new ByteArrayRef();

    ref.setData(bytes);

    // initialize the lazy object
    lazyObject.init(ref, 0, ref.getData().length);

    return lazyObject;
  }
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.ByteArrayRef

    boolean inited;
    boolean fieldSkipped;

    public FieldInfo(LazyObject lazyObject, boolean fieldSkipped) {
      field = lazyObject;
      cachedByteArrayRef = new ByteArrayRef();
      if (fieldSkipped) {
        this.fieldSkipped = true;
        inited = true;
      } else {
        inited = false;
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.