Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

Package org.apache.hadoop.hive.serde2.lazy

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector
LazySimpleSerDe can be used to read the same data format as MetadataTypedColumnsetSerDe and TCTLSeparatedProtocol. However, LazySimpleSerDe creates Objects in a lazy way, to provide better performance. Also LazySimpleSerDe outputs typed columns instead of treating all columns as String like MetadataTypedColumnsetSerDe.

        }
      }
      String names = namesSb.toString();
      String types = typesSb.toString();


      serde = new LazySimpleSerDe();
      Properties props = new Properties();
      if (names.length() > 0) {
        LOG.debug("Column names: " + names);
        props.setProperty(serdeConstants.LIST_COLUMNS, names);
      }

View Full Code Here

                   StreamFactory writer,
                   boolean nullable, Configuration conf,
                   boolean useVInts, boolean lowMemoryMode,
                   MemoryEstimate memoryEstimate) throws IOException {
      super(columnId, inspector, writer, nullable, conf, useVInts, memoryEstimate);
      ListObjectInspector listObjectInspector = (ListObjectInspector) inspector;
      childrenWriters = new TreeWriter[1];
      childrenWriters[0] =
        createTreeWriter(listObjectInspector.getListElementObjectInspector(),
          writer, true, conf, useVInts, lowMemoryMode, memoryEstimate);
      lengths =
        new RunLengthIntegerWriter(writer.createStream(columnId,
            OrcProto.Stream.Kind.LENGTH), false, INT_BYTE_SIZE, useVInts);
      recordPosition(rowIndexPosition);

View Full Code Here


    @Override
    void write(Object obj) throws IOException {
      long rawDataSize = 0;
      if (obj != null) {
        ListObjectInspector insp = (ListObjectInspector) inspector;
        int len = insp.getListLength(obj);
        lengths.write(len);
        for(int i=0; i < len; ++i) {
          childrenWriters[0].write(insp.getListElement(obj, i));
          rawDataSize += childrenWriters[0].getRowRawDataSize();
        }
      }
      super.write(obj, rawDataSize);
    }

View Full Code Here

    if (o == null || o.getClass() != getClass()) {
      return false;
    } else if (o == this) {
      return true;
    } else {
      ObjectInspector other = ((OrcLazyListObjectInspector) o).child;
      return other.equals(child);
    }
  }

View Full Code Here

  /**
   * Tests a writing a stripe with a string column, which enters low memory mode before the second
   * index stride is complete, and does not complete that stride.
   */
  public void testStringEnterLowMemoryModeInSecondStride() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (StringStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }

View Full Code Here

  /**
   * Tests a writing a stripe with an int column, which enters low memory mode before the second
   * index stride is complete, and does not complete that stride.
   */
  public void testIntEnterLowMemoryModeInSecondStride() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (IntStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }

View Full Code Here

  /**
   * Tests a writing a stripe with a string column, which enters low memory mode just before the
   * second stride starts
   */
  public void testStringEnterLowMemoryModeAtStrideStart() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (StringStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }

View Full Code Here

  /**
   * Tests a writing a stripe with an int column, which enters low memory mode just before the
   * second stride starts
   */
  public void testIntEnterLowMemoryModeAtStrideStart() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (IntStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }

View Full Code Here

  /**
   * Tests a writing a stripe with a string column, which enters low memory mode just after the
   * second stride starts
   */
  public void testStringEnterLowMemoryModeAfterStrideStart() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (StringStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }

View Full Code Here

  /**
   * Tests a writing a stripe with an int column, which enters low memory mode just after the
   * second stride starts
   */
  public void testIntEnterLowMemoryModeAfterStrideStart() throws Exception {
    ObjectInspector inspector;
    synchronized (TestOrcFile.class) {
      inspector = ObjectInspectorFactory.getReflectionObjectInspector
          (IntStruct.class,
              ObjectInspectorFactory.ObjectInspectorOptions.JAVA);
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

com.asakusafw.directio.hive.orc.OrcFileInput

com.asakusafw.directio.hive.serde.DataModelDriver

com.asakusafw.directio.hive.serde.DataModelDriverTest

com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest

com.facebook.hive.orc.lazy.LazyDoubleTreeReader

com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader

com.facebook.hive.orc.lazy.LazyShortDirectTreeReader

com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector

com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter

com.facebook.hive.orc.OrcStruct$OrcStructInspector

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.