Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

Package org.apache.hadoop.hive.serde2.lazy

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector
LazySimpleSerDe can be used to read the same data format as MetadataTypedColumnsetSerDe and TCTLSeparatedProtocol. However, LazySimpleSerDe creates Objects in a lazy way, to provide better performance. Also LazySimpleSerDe outputs typed columns instead of treating all columns as String like MetadataTypedColumnsetSerDe.

   * Tests writing a stripe with a string column, which doesn't do dictionary encoding, then
   * re-evaluates whether it should do dictionary encoding or not.  While it's re-evaluating, it
   * enters low memory mode.
   */
  public void testStringEnterLowMemoryModeAndOnNotCarriedOverStripe() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (StringStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }

View Full Code Here

   * Tests writing a stripe with an int column, which doesn't do dictionary encoding, then
   * re-evaluates whether it should do dictionary encoding or not.  While it's re-evaluating, it
   * enters low memory mode
   */
  public void testIntegerEnterLowMemoryModeAndOnNotCarriedOverStripe() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (IntStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }

View Full Code Here

  @Test
  /**
   * Tests calling seekToRow to make sure it updates the stripe accordingly
   */
  public void testSeekToRow() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (IntStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }

View Full Code Here

  /**
   * Tests that when a reader is initialized using offset, length the stripes included are
   * those that start in the range [offset, offset + length)
   */
  public void testSplitStripe() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (IntStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }

View Full Code Here

        return struct;
      }
    };


    // Test control case (cases match)
    StructField field = STRUCT_OI.getStructFieldRef(FIELD_0);
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            STRUCT_OI.getStructFieldData(struct, field)));
    // Test upper case
    field = STRUCT_OI.getStructFieldRef(FIELD_0.toUpperCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            STRUCT_OI.getStructFieldData(struct, field)));
    // Test lower case (even if someone changes the value of FIELD_0 in the future either upper
    // or lower case should be different from the actual case)
    field = STRUCT_OI.getStructFieldRef(FIELD_0.toLowerCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            STRUCT_OI.getStructFieldData(struct, field)));
  }

View Full Code Here

  public void testCaseInsensitiveFieldsStruct() throws Exception {
    OrcStruct struct = new OrcStruct(Lists.newArrayList(FIELD_0));
    struct.setFieldValue(0, new Text("a"));


    // Test control case (cases match)
    StructField field = NON_LAZY_STRUCT_OI.getStructFieldRef(FIELD_0);
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            NON_LAZY_STRUCT_OI.getStructFieldData(struct, field)));
    // Test upper case
    field = NON_LAZY_STRUCT_OI.getStructFieldRef(FIELD_0.toUpperCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            NON_LAZY_STRUCT_OI.getStructFieldData(struct, field)));
    // Test lower case (even if someone changes the value of FIELD_0 in the future either upper
    // or lower case should be different from the actual case)
    field = NON_LAZY_STRUCT_OI.getStructFieldRef(FIELD_0.toLowerCase());
    Assert.assertEquals("a",
        ((StringObjectInspector) field.getFieldObjectInspector()).getPrimitiveJavaObject(
            NON_LAZY_STRUCT_OI.getStructFieldData(struct, field)));
  }

View Full Code Here

    writer.close();
    OrcConf.setIntVar(conf, OrcConf.ConfVars.HIVE_ORC_READ_COMPRESSION_STRIDES, 2);
    OrcConf.setBoolVar(conf, OrcConf.ConfVars.HIVE_ORC_EAGER_HDFS_READ, false);
    Reader reader = OrcFile.createReader(fs, testFilePath, conf);


    StructObjectInspector readerInspector = (StructObjectInspector) reader.getObjectInspector();
    List<? extends StructField> fields = readerInspector.getAllStructFieldRefs();
    DoubleObjectInspector columnInspector =
        (DoubleObjectInspector) fields.get(0).getFieldObjectInspector();


    RecordReader rows = reader.rows(null);
    Object row = null;


    // Skip enough values to get to the 2nd index stride in the first chunk
    for (int i = 0; i < 40001; i++) {
      row = rows.next(row);
    }


    // This will set previousOffset to be the size of the first compression block and the
    // compressionOffset to some other value (doesn't matter what point is it's different from the
    // start of the compression block)
    assertEquals(values[40000], columnInspector.get(readerInspector.getStructFieldData(row,
        fields.get(0))));


    // Skip enough values to get to the 2nd index stride of the second chunk
    for (int i = 0; i < 80000; i++) {
      rows.next(row);
    }


    // When seek is called, previousOffset will equal newCompressedOffset since the former is the
    // the length of the first compression block and the latter is the length of the third
    // compression block (remember the chunks contain 2 index strides), so if we only check this
    // (or for some other reason) we will not adjust compressedIndex, we will read the wrong data
    assertEquals(values[120000], columnInspector.get(readerInspector.getStructFieldData(row, fields.get(0))));


    rows.close();
  }

View Full Code Here

        stats[7].toString());


    assertEquals("count: 2 min: bye max: hi", stats[9].toString());


    // check the inspectors
    StructObjectInspector readerInspector =
        (StructObjectInspector) reader.getObjectInspector();
    assertEquals(ObjectInspector.Category.STRUCT,
        readerInspector.getCategory());
    assertEquals("struct<boolean1:boolean,byte1:tinyint,short1:smallint,"
        + "int1:int,long1:bigint,float1:float,double1:double,bytes1:"
        + "binary,string1:string,middle:struct<list:array<struct<int1:int,"
        + "string1:string>>>,list:array<struct<int1:int,string1:string>>,"
        + "map:map<string,struct<int1:int,string1:string>>>",
        readerInspector.getTypeName());
    List<? extends StructField> fields =
        readerInspector.getAllStructFieldRefs();
    BooleanObjectInspector bo = (BooleanObjectInspector) readerInspector.
        getStructFieldRef("boolean1").getFieldObjectInspector();
    ByteObjectInspector by = (ByteObjectInspector) readerInspector.
        getStructFieldRef("byte1").getFieldObjectInspector();
    ShortObjectInspector sh = (ShortObjectInspector) readerInspector.
        getStructFieldRef("short1").getFieldObjectInspector();
    IntObjectInspector in = (IntObjectInspector) readerInspector.
        getStructFieldRef("int1").getFieldObjectInspector();
    LongObjectInspector lo = (LongObjectInspector) readerInspector.
        getStructFieldRef("long1").getFieldObjectInspector();
    FloatObjectInspector fl = (FloatObjectInspector) readerInspector.
        getStructFieldRef("float1").getFieldObjectInspector();
    DoubleObjectInspector dbl = (DoubleObjectInspector) readerInspector.
        getStructFieldRef("double1").getFieldObjectInspector();
    BinaryObjectInspector bi = (BinaryObjectInspector) readerInspector.
        getStructFieldRef("bytes1").getFieldObjectInspector();
    StringObjectInspector st = (StringObjectInspector) readerInspector.
        getStructFieldRef("string1").getFieldObjectInspector();
    StructObjectInspector mid = (StructObjectInspector) readerInspector.
        getStructFieldRef("middle").getFieldObjectInspector();
    List<? extends StructField> midFields =
        mid.getAllStructFieldRefs();
    ListObjectInspector midli =
        (ListObjectInspector) midFields.get(0).getFieldObjectInspector();
    StructObjectInspector inner = (StructObjectInspector)
        midli.getListElementObjectInspector();
    List<? extends StructField> inFields = inner.getAllStructFieldRefs();
    IntObjectInspector inner_in = (IntObjectInspector) inFields.get(0).getFieldObjectInspector();
    StringObjectInspector inner_st = (StringObjectInspector) inFields.get(1).getFieldObjectInspector();
    ListObjectInspector li = (ListObjectInspector) readerInspector.
        getStructFieldRef("list").getFieldObjectInspector();
    MapObjectInspector ma = (MapObjectInspector) readerInspector.
        getStructFieldRef("map").getFieldObjectInspector();
    StructObjectInspector lc = (StructObjectInspector)
        li.getListElementObjectInspector();
    StringObjectInspector mk = (StringObjectInspector)
        ma.getMapKeyObjectInspector();
    StructObjectInspector mv = (StructObjectInspector)
        ma.getMapValueObjectInspector();
    RecordReader rows = reader.rows(null);
    Object row = rows.next(null);
    assertNotNull(row);
    // check the contents of the first row

View Full Code Here

                     StreamFactory writer,
                     boolean nullable, Configuration conf,
                     boolean useVInts, boolean lowMemoryMode,
                     MemoryEstimate memoryEstimate) throws IOException {
      super(columnId, inspector, writer, nullable, conf, useVInts, memoryEstimate);
      StructObjectInspector structObjectInspector =
        (StructObjectInspector) inspector;
      fields = structObjectInspector.getAllStructFieldRefs();
      childrenWriters = new TreeWriter[fields.size()];
      for(int i=0; i < childrenWriters.length; ++i) {
        childrenWriters[i] = createTreeWriter(
          fields.get(i).getFieldObjectInspector(), writer, true, conf, useVInts,
          lowMemoryMode, memoryEstimate);

View Full Code Here


    @Override
    void write(Object obj) throws IOException {
      long rawDataSize = 0;
      if (obj != null) {
        StructObjectInspector insp = (StructObjectInspector) inspector;
        List<Object> fieldDataList = insp.getStructFieldsDataAsList(obj);


        for(int i = 0; i < fields.size(); ++i) {
          TreeWriter writer = childrenWriters[i];
          writer.write(fieldDataList.get(i));
          rawDataSize += writer.getRowRawDataSize();

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

com.asakusafw.directio.hive.orc.OrcFileInput

com.asakusafw.directio.hive.serde.DataModelDriver

com.asakusafw.directio.hive.serde.DataModelDriverTest

com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest

com.facebook.hive.orc.lazy.LazyDoubleTreeReader

com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader

com.facebook.hive.orc.lazy.LazyShortDirectTreeReader

com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector

com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter

com.facebook.hive.orc.OrcStruct$OrcStructInspector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.