/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hcatalog.data;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import junit.framework.Assert;
import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe;
import org.apache.hadoop.io.Writable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class TestHCatRecordSerDe extends TestCase {
private static final Logger LOG = LoggerFactory.getLogger(TestHCatRecordSerDe.class);
public Map<Properties, HCatRecord> getData() {
Map<Properties, HCatRecord> data = new HashMap<Properties, HCatRecord>();
List<Object> rlist = new ArrayList<Object>(11);
rlist.add(new Byte("123"));
rlist.add(new Short("456"));
rlist.add(new Integer(789));
rlist.add(new Long(1000L));
rlist.add(new Double(5.3D));
rlist.add(new Float(2.39F));
rlist.add(new String("hcat and hadoop"));
rlist.add(null);
List<Object> innerStruct = new ArrayList<Object>(2);
innerStruct.add(new String("abc"));
innerStruct.add(new String("def"));
rlist.add(innerStruct);
List<Integer> innerList = new ArrayList<Integer>();
innerList.add(314);
innerList.add(007);
rlist.add(innerList);
Map<Short, String> map = new HashMap<Short, String>(3);
map.put(new Short("2"), "hcat is cool");
map.put(new Short("3"), "is it?");
map.put(new Short("4"), "or is it not?");
rlist.add(map);
rlist.add(new Boolean(true));
List<Object> c1 = new ArrayList<Object>();
List<Object> c1_1 = new ArrayList<Object>();
c1_1.add(new Integer(12));
List<Object> i2 = new ArrayList<Object>();
List<Integer> ii1 = new ArrayList<Integer>();
ii1.add(new Integer(13));
ii1.add(new Integer(14));
i2.add(ii1);
Map<String, List<?>> ii2 = new HashMap<String, List<?>>();
List<Integer> iii1 = new ArrayList<Integer>();
iii1.add(new Integer(15));
ii2.put("phew", iii1);
i2.add(ii2);
c1_1.add(i2);
c1.add(c1_1);
rlist.add(c1);
List<Object> am = new ArrayList<Object>();
Map<String, String> am_1 = new HashMap<String, String>();
am_1.put("noo", "haha");
am.add(am_1);
rlist.add(am);
List<Object> aa = new ArrayList<Object>();
List<String> aa_1 = new ArrayList<String>();
aa_1.add("bloo");
aa_1.add("bwahaha");
aa.add(aa_1);
rlist.add(aa);
String typeString =
"tinyint,smallint,int,bigint,double,float,string,string,"
+ "struct<a:string,b:string>,array<int>,map<smallint,string>,boolean,"
+ "array<struct<i1:int,i2:struct<ii1:array<int>,ii2:map<string,struct<iii1:int>>>>>,"
+ "array<map<string,string>>,array<array<string>>";
Properties props = new Properties();
props.put(serdeConstants.LIST_COLUMNS, "ti,si,i,bi,d,f,s,n,r,l,m,b,c1,am,aa");
props.put(serdeConstants.LIST_COLUMN_TYPES, typeString);
// props.put(Constants.SERIALIZATION_NULL_FORMAT, "\\N");
// props.put(Constants.SERIALIZATION_FORMAT, "1");
data.put(props, new DefaultHCatRecord(rlist));
return data;
}
public void testRW() throws Exception {
Configuration conf = new Configuration();
for (Entry<Properties, HCatRecord> e : getData().entrySet()) {
Properties tblProps = e.getKey();
HCatRecord r = e.getValue();
HCatRecordSerDe hrsd = new HCatRecordSerDe();
hrsd.initialize(conf, tblProps);
LOG.info("ORIG: {}", r);
Writable s = hrsd.serialize(r, hrsd.getObjectInspector());
LOG.info("ONE: {}", s);
HCatRecord r2 = (HCatRecord) hrsd.deserialize(s);
Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r, r2));
// If it went through correctly, then s is also a HCatRecord,
// and also equal to the above, and a deepcopy, and this holds
// through for multiple levels more of serialization as well.
Writable s2 = hrsd.serialize(s, hrsd.getObjectInspector());
LOG.info("TWO: {}", s2);
Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) s));
Assert.assertTrue(HCatDataCheckUtil.recordsEqual(r, (HCatRecord) s2));
// serialize using another serde, and read out that object repr.
LazySimpleSerDe testSD = new LazySimpleSerDe();
testSD.initialize(conf, tblProps);
Writable s3 = testSD.serialize(s, hrsd.getObjectInspector());
LOG.info("THREE: {}", s3);
Object o3 = testSD.deserialize(s3);
Assert.assertFalse(r.getClass().equals(o3.getClass()));
// then serialize again using hrsd, and compare results
HCatRecord s4 = (HCatRecord) hrsd.serialize(o3, testSD.getObjectInspector());
LOG.info("FOUR: {}", s4);
// Test LazyHCatRecord init and read
LazyHCatRecord s5 = new LazyHCatRecord(o3, testSD.getObjectInspector());
LOG.info("FIVE: {}", s5);
LazyHCatRecord s6 = new LazyHCatRecord(s4, hrsd.getObjectInspector());
LOG.info("SIX: {}", s6);
}
}
}