Examples of LazySimpleSerDe

org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
LazySimpleSerDe can be used to read the same data format as MetadataTypedColumnsetSerDe and TCTLSeparatedProtocol. However, LazySimpleSerDe creates Objects in a lazy way, to provide better performance. Also LazySimpleSerDe outputs typed columns instead of treating all columns as String like MetadataTypedColumnsetSerDe.

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

    // empty maps
    TextStringMapHolder o1 = new TextStringMapHolder();
    StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(TextStringMapHolder.class, ObjectInspectorOptions.JAVA);


    LazySimpleSerDe serde = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    Properties tbl = new Properties();
    tbl.setProperty(serdeConstants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
    SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl,
        LazySimpleSerDe.class.getName());
    serde.initialize(conf, tbl);
    ObjectInspector oi2 = serde.getObjectInspector();


    Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);


    int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
    assertEquals(0, rc);

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

    // empty maps
    StringTextMapHolder o1 = new StringTextMapHolder();
    StructObjectInspector oi1 = (StructObjectInspector) ObjectInspectorFactory
        .getReflectionObjectInspector(StringTextMapHolder.class, ObjectInspectorOptions.JAVA);


    LazySimpleSerDe serde = new LazySimpleSerDe();
    Configuration conf = new Configuration();
    Properties tbl = new Properties();
    tbl.setProperty(serdeConstants.LIST_COLUMNS, ObjectInspectorUtils.getFieldNames(oi1));
    tbl.setProperty(serdeConstants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi1));
    SerDeParameters serdeParams = LazySimpleSerDe.initSerdeParams(conf, tbl,
        LazySimpleSerDe.class.getName());
    serde.initialize(conf, tbl);
    ObjectInspector oi2 = serde.getObjectInspector();


    Object o2 = serializeAndDeserialize(o1, oi1, serde, serdeParams);


    int rc = ObjectInspectorUtils.compare(o1, oi1, o2, oi2, new SimpleMapEqualComparer());
    assertEquals(0, rc);

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

        }
      }
      String names = namesSb.toString();
      String types = typesSb.toString();


      serde = new LazySimpleSerDe();
      Properties props = new Properties();
      if (names.length() > 0) {
        LOG.info("Column names: " + names);
        props.setProperty(Constants.LIST_COLUMNS, names);
      }

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

  protected LazySimpleSerDe createSerde(Table tbl, HiveConf conf)
          throws SerializationError {
    try {
      Properties tableProps = MetaStoreUtils.getTableMetadata(tbl);
      tableProps.setProperty("field.delim", String.valueOf(serdeSeparator));
      LazySimpleSerDe serde = new LazySimpleSerDe();
      SerDeUtils.initializeSerDe(serde, conf, tableProps, null);
      return serde;
    } catch (SerDeException e) {
      throw new SerializationError("Error initializing serde", e);
    }

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

      LOG.info("TWO: {}", s2);
      Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) s));
      Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) s2));


      // serialize using another serde, and read out that object repr.
      LazySimpleSerDe testSD = new LazySimpleSerDe();
      SerDeUtils.initializeSerDe(testSD, conf, tblProps, null);


      Writable s3 = testSD.serialize(s, hrsd.getObjectInspector());
      LOG.info("THREE: {}", s3);
      Object o3 = testSD.deserialize(s3);
      Assert.assertFalse(r.getClass().equals(o3.getClass()));


      // then serialize again using hrsd, and compare results
      HCatRecord s4 = (HCatRecord) hrsd.serialize(o3, testSD.getObjectInspector());
      LOG.info("FOUR: {}", s4);


      // Test LazyHCatRecord init and read
      LazyHCatRecord s5 = new LazyHCatRecord(o3, testSD.getObjectInspector());
      LOG.info("FIVE: {}", s5);


      LazyHCatRecord s6 = new LazyHCatRecord(s4, hrsd.getObjectInspector());
      LOG.info("SIX: {}", s6);

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

            if(file.isDirectory()) {
                for(File f : file.listFiles()) {
                    count += loadPredictionModel(label2model, f, labelOI, featureOI, weightOI);
                }
            } else {
                LazySimpleSerDe serde = HiveUtils.getLineSerde(labelOI, featureOI, weightOI);
                StructObjectInspector lineOI = (StructObjectInspector) serde.getObjectInspector();
                StructField c1ref = lineOI.getStructFieldRef("c1");
                StructField c2ref = lineOI.getStructFieldRef("c2");
                StructField c3ref = lineOI.getStructFieldRef("c3");
                PrimitiveObjectInspector c1refOI = (PrimitiveObjectInspector) c1ref.getFieldObjectInspector();
                PrimitiveObjectInspector c2refOI = (PrimitiveObjectInspector) c2ref.getFieldObjectInspector();
                FloatObjectInspector c3refOI = (FloatObjectInspector) c3ref.getFieldObjectInspector();


                BufferedReader reader = null;
                try {
                    reader = HadoopUtils.getBufferedReader(file);
                    String line;
                    while((line = reader.readLine()) != null) {
                        count++;
                        Text lineText = new Text(line);
                        Object lineObj = serde.deserialize(lineText);
                        List<Object> fields = lineOI.getStructFieldsDataAsList(lineObj);
                        Object f0 = fields.get(0);
                        Object f1 = fields.get(1);
                        Object f2 = fields.get(2);
                        if(f0 == null || f1 == null || f2 == null) {

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

            if(file.isDirectory()) {
                for(File f : file.listFiles()) {
                    count += loadPredictionModel(label2model, f, labelOI, featureOI, weightOI, covarOI);
                }
            } else {
                LazySimpleSerDe serde = HiveUtils.getLineSerde(labelOI, featureOI, weightOI, covarOI);
                StructObjectInspector lineOI = (StructObjectInspector) serde.getObjectInspector();
                StructField c1ref = lineOI.getStructFieldRef("c1");
                StructField c2ref = lineOI.getStructFieldRef("c2");
                StructField c3ref = lineOI.getStructFieldRef("c3");
                StructField c4ref = lineOI.getStructFieldRef("c4");
                PrimitiveObjectInspector c1refOI = (PrimitiveObjectInspector) c1ref.getFieldObjectInspector();
                PrimitiveObjectInspector c2refOI = (PrimitiveObjectInspector) c2ref.getFieldObjectInspector();
                FloatObjectInspector c3refOI = (FloatObjectInspector) c3ref.getFieldObjectInspector();
                FloatObjectInspector c4refOI = (FloatObjectInspector) c4ref.getFieldObjectInspector();


                BufferedReader reader = null;
                try {
                    reader = HadoopUtils.getBufferedReader(file);
                    String line;
                    while((line = reader.readLine()) != null) {
                        count++;
                        Text lineText = new Text(line);
                        Object lineObj = serde.deserialize(lineText);
                        List<Object> fields = lineOI.getStructFieldsDataAsList(lineObj);
                        Object f0 = fields.get(0);
                        Object f1 = fields.get(1);
                        Object f2 = fields.get(2);
                        Object f3 = fields.get(3);

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

        return STRING_TYPE_NAME.equals(typeName);
    }


    public static LazySimpleSerDe getKeyValueLineSerde(PrimitiveObjectInspector keyOI, PrimitiveObjectInspector valueOI)
            throws SerDeException {
        LazySimpleSerDe serde = new LazySimpleSerDe();
        Configuration conf = new Configuration();
        Properties tbl = new Properties();
        tbl.setProperty("columns", "key,value");
        tbl.setProperty("columns.types", keyOI.getTypeName() + "," + valueOI.getTypeName());
        serde.initialize(conf, tbl);
        return serde;
    }

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

    public static LazySimpleSerDe getLineSerde(PrimitiveObjectInspector... OIs)
            throws SerDeException {
        if(OIs.length == 0) {
            throw new IllegalArgumentException("OIs must be specified");
        }
        LazySimpleSerDe serde = new LazySimpleSerDe();
        Configuration conf = new Configuration();
        Properties tbl = new Properties();


        StringBuilder columnNames = new StringBuilder();
        StringBuilder columnTypes = new StringBuilder();
        for(int i = 0; i < OIs.length; i++) {
            columnNames.append('c').append(i + 1).append(',');
            columnTypes.append(OIs[i].getTypeName()).append(',');
        }
        columnNames.deleteCharAt(columnNames.length() - 1);
        columnTypes.deleteCharAt(columnTypes.length() - 1);


        tbl.setProperty("columns", columnNames.toString());
        tbl.setProperty("columns.types", columnTypes.toString());
        serde.initialize(conf, tbl);
        return serde;
    }

View Full Code Here

Examples of org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe

            if(file.isDirectory()) {
                for(File f : file.listFiles()) {
                    count += loadPredictionModel(model, f, keyOI, valueOI);
                }
            } else {
                LazySimpleSerDe serde = HiveUtils.getKeyValueLineSerde(keyOI, valueOI);
                StructObjectInspector lineOI = (StructObjectInspector) serde.getObjectInspector();
                StructField keyRef = lineOI.getStructFieldRef("key");
                StructField valueRef = lineOI.getStructFieldRef("value");
                PrimitiveObjectInspector keyRefOI = (PrimitiveObjectInspector) keyRef.getFieldObjectInspector();
                FloatObjectInspector varRefOI = (FloatObjectInspector) valueRef.getFieldObjectInspector();


                BufferedReader reader = null;
                try {
                    reader = HadoopUtils.getBufferedReader(file);
                    String line;
                    while((line = reader.readLine()) != null) {
                        count++;
                        Text lineText = new Text(line);
                        Object lineObj = serde.deserialize(lineText);
                        List<Object> fields = lineOI.getStructFieldsDataAsList(lineObj);
                        Object f0 = fields.get(0);
                        Object f1 = fields.get(1);
                        if(f0 == null || f1 == null) {
                            continue; // avoid the case that key or value is null

View Full Code Here

0 1 2 3 4

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.