Examples of LazySimpleSerDe

org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
LazySimpleSerDe can be used to read the same data format as MetadataTypedColumnsetSerDe and TCTLSeparatedProtocol. However, LazySimpleSerDe creates Objects in a lazy way, to provide better performance. Also LazySimpleSerDe outputs typed columns instead of treating all columns as String like MetadataTypedColumnsetSerDe.

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

    // empty maps
    StringTextMapHolder o1 = new StringTextMapHolder();
    StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(StringTextMapHolder.class, ObjectInspectorOptions.JAVA);


    LazySimpleSerDe serde = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    Properties tbl = new Properties();
    tbl.setProperty(Constants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
    tbl.setProperty(Constants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
    SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl,
        LazySimpleSerDe.class.getName());
    serde.initialize(conf, tbl);
    ObjectInspector oi2 = serde.getObjectInspector();


    Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);


    int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer());
    assertEquals(0, rc);

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

    // empty maps
    TextStringMapHolder o1 = new TextStringMapHolder();
    StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(TextStringMapHolder.class, ObjectInspectorOptions.JAVA);


    LazySimpleSerDe serde = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    Properties tbl = new Properties();
    tbl.setProperty(Constants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
    tbl.setProperty(Constants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
    SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl,
        LazySimpleSerDe.class.getName());
    serde.initialize(conf, tbl);
    ObjectInspector oi2 = serde.getObjectInspector();


    Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);


    int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
    assertEquals(0, rc);

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

        }
      }
      String names = namesSb.toString();
      String types = typesSb.toString();


      serde = new LazySimpleSerDe();
      Properties props = new Properties();
      if (names.length() > 0) {
        LOG.debug("Column names: " + names);
        props.setProperty(Constants.LIST_COLUMNS, names);
      }

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

      System.out.println("TWO:"+s2.toString());
      Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r,(HCatRecord)s));
      Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r,(HCatRecord)s2));
      
      // serialize using another serde, and read out that object repr.
      LazySimpleSerDe testSD = new LazySimpleSerDe();
      testSD.initialize(conf, tblProps);


      Writable s3 = testSD.serialize(s, hrsd.getObjectInspector());
      System.out.println("THREE:"+s3.toString());
      Object o3 = testSD.deserialize(s3);
      Assert.assertFalse(r.getClass().equals(o3.getClass()));


      // then serialize again using hrsd, and compare results
      HCatRecord s4 = (HCatRecord) hrsd.serialize(o3, testSD.getObjectInspector());
      System.out.println("FOUR:"+s4.toString());


      // Test LazyHCatRecord init and read
      LazyHCatRecord s5 = new LazyHCatRecord(o3,testSD.getObjectInspector());
      System.out.println("FIVE:"+s5.toString());


      LazyHCatRecord s6 = new LazyHCatRecord(s4,hrsd.getObjectInspector());
      System.out.println("SIX:"+s6.toString());

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

    {
      Properties p = new Properties();
      p.put(Constants.FIELD_DELIM, fieldDelim);
      p.put(Constants.LIST_COLUMNS, columns);
      p.put(Constants.LIST_COLUMN_TYPES, types);
      LazySimpleSerDe sd = new LazySimpleSerDe();
      sd.initialize(null, p);
      return sd;
    }
    catch (SerDeException se)
    {
      throw new BaseException(se);

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

      LOG.info("TWO: {}", s2);
      Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) s));
      Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) s2));


      // serialize using another serde, and read out that object repr.
      LazySimpleSerDe testSD = new LazySimpleSerDe();
      testSD.initialize(conf, tblProps);


      Writable s3 = testSD.serialize(s, hrsd.getObjectInspector());
      LOG.info("THREE: {}", s3);
      Object o3 = testSD.deserialize(s3);
      Assert.assertFalse(r.getClass().equals(o3.getClass()));


      // then serialize again using hrsd, and compare results
      HCatRecord s4 = (HCatRecord) hrsd.serialize(o3, testSD.getObjectInspector());
      LOG.info("FOUR: {}", s4);


      // Test LazyHCatRecord init and read
      LazyHCatRecord s5 = new LazyHCatRecord(o3, testSD.getObjectInspector());
      LOG.info("FIVE: {}", s5);


      LazyHCatRecord s6 = new LazyHCatRecord(s4, hrsd.getObjectInspector());
      LOG.info("SIX: {}", s6);

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

      LOG.info("TWO: {}", s2);
      Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) s));
      Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) s2));


      // serialize using another serde, and read out that object repr.
      LazySimpleSerDe testSD = new LazySimpleSerDe();
      testSD.initialize(conf, tblProps);


      Writable s3 = testSD.serialize(s, hrsd.getObjectInspector());
      LOG.info("THREE: {}", s3);
      Object o3 = testSD.deserialize(s3);
      Assert.assertFalse(r.getClass().equals(o3.getClass()));


      // then serialize again using hrsd, and compare results
      HCatRecord s4 = (HCatRecord) hrsd.serialize(o3, testSD.getObjectInspector());
      LOG.info("FOUR: {}", s4);


      // Test LazyHCatRecord init and read
      LazyHCatRecord s5 = new LazyHCatRecord(o3, testSD.getObjectInspector());
      LOG.info("FIVE: {}", s5);


      LazyHCatRecord s6 = new LazyHCatRecord(s4, hrsd.getObjectInspector());
      LOG.info("SIX: {}", s6);

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe


  public void testLazySimpleSerDe() throws Throwable {
    try {
      // Create the SerDe
      System.out.println("test: testLazySimpleSerDe");
      LazySimpleSerDe serDe = new LazySimpleSerDe();
      Configuration conf = new Configuration();
      Properties tbl = createProperties();
      serDe.initialize(conf, tbl);


      // Data
      Text t = new Text("123\t456\t789\t1000\t5.3\thive and hadoop\t1.\tNULL");
      // Test
      deserializeAndSerializeLazySimple(serDe, t);

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

    // empty maps
    TextStringMapHolder o1 = new TextStringMapHolder();
    StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(TextStringMapHolder.class, ObjectInspectorOptions.JAVA);


    LazySimpleSerDe serde = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    Properties tbl = new Properties();
    tbl.setProperty(serdeConstants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
    SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl,
        LazySimpleSerDe.class.getName());
    serde.initialize(conf, tbl);
    ObjectInspector oi2 = serde.getObjectInspector();


    Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);


    int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer());
    assertEquals(0, rc);

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

    // empty maps
    StringTextMapHolder o1 = new StringTextMapHolder();
    StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(StringTextMapHolder.class, ObjectInspectorOptions.JAVA);


    LazySimpleSerDe serde = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    Properties tbl = new Properties();
    tbl.setProperty(serdeConstants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
    SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl,
        LazySimpleSerDe.class.getName());
    serde.initialize(conf, tbl);
    ObjectInspector oi2 = serde.getObjectInspector();


    Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);


    int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new CrossMapEqualComparer());
    assertEquals(0, rc);

View Full Code Here

0 1 2 3 4

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.