JobConf hdpConf = HdpBootstrap.hadoopConfig();
hdpConf.set(ConfigurationOptions.ES_RESOURCE, target);
//JavaPairRDD data = sc.newAPIHadoopRDD(hdpConf, EsInputFormat.class, NullWritable.class, MapWritable.class);
JavaPairRDD data = sc.hadoopRDD(hdpConf, EsInputFormat.class, NullWritable.class, MapWritable.class);
long messages = data.filter(new Function<Tuple2<Text, MapWritable>, Boolean>() {
public Boolean call(Tuple2<Text, MapWritable> t) { return t._2.containsKey(new Text("message")); }
}).count();
JavaRDD map = data.map(new Function<Tuple2<Text, MapWritable>, Map<String, Object>>() {
public Map<String, Object> call(Tuple2<Text, MapWritable> v1) throws Exception {
return (Map<String, Object>) WritableUtils.fromWritable(v1._2);
}
});
JavaRDD fooBar = data.map(new Function<Tuple2<Text, MapWritable>, String>() {
public String call(Tuple2<Text, MapWritable> v1) throws Exception {
return v1._1.toString();
}
});
assertThat((int) data.count(), is(2));
System.out.println(data.take(10));
System.out.println(messages);
System.out.println(fooBar.take(2));
System.out.println(map.take(10));
}